diff options
78 files changed, 1626 insertions, 408 deletions
diff --git a/.bumpversion.cfg b/.bumpversion.cfg index fcaf46306..051e07b87 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.0.0 +current_version = 3.0.2 commit = True message = Release v{new_version} tag = True diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d7963ea7..c2babf437 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.10' - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: '16.x' if: "matrix.testenv == 'eslint'" @@ -157,28 +157,28 @@ jobs: - testenv: py310-pyqt65 os: ubuntu-22.04 python: "3.10" - ### PyQt 6.5 (Python 3.11) - - testenv: py311-pyqt65 + ### PyQt 6.6 (Python 3.11) + - testenv: py311-pyqt66 os: ubuntu-22.04 python: "3.11" - ### PyQt 6.5 (Python 3.12) - - testenv: py312-pyqt65 + ### PyQt 6.6 (Python 3.12) + - testenv: py312-pyqt66 os: ubuntu-22.04 - python: "3.12-dev" - ### macOS Big Sur: PyQt 5.15 (Python 3.9 to match PyInstaller env) - - testenv: py39-pyqt515 + python: "3.12" + ### macOS Big Sur + - testenv: py311-pyqt66 os: macos-11 - python: "3.9" + python: "3.11" args: "tests/unit" # Only run unit tests on macOS ### macOS Monterey - - testenv: py39-pyqt515 + - testenv: py311-pyqt66 os: macos-12 - python: "3.9" + python: "3.11" args: "tests/unit" # Only run unit tests on macOS - ### Windows: PyQt 5.15 (Python 3.9 to match PyInstaller env) - - testenv: py39-pyqt515 + ### Windows + - testenv: py311-pyqt66 os: windows-2019 - python: "3.9" + python: "3.11" runs-on: "${{ matrix.os }}" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 76332e8ba..433cd3c0b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -15,24 +15,19 @@ jobs: matrix: include: - os: macos-11 - branch: main toxenv: build-release-qt5 name: qt5-macos - os: windows-2019 - branch: main toxenv: build-release-qt5 name: qt5-windows - os: macos-11 args: --debug - branch: main toxenv: build-release-qt5 name: qt5-macos-debug - os: windows-2019 args: --debug - branch: main toxenv: build-release-qt5 name: qt5-windows-debug - - os: macos-11 toxenv: build-release name: macos @@ -52,7 +47,6 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: "${{ matrix.branch }}" persist-credentials: false - name: Set up Python uses: actions/setup-python@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a378e53d9..fd3bc5cd8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,7 +34,7 @@ jobs: contents: write # To push release commit/tag steps: - name: Find release branch - uses: actions/github-script@v6 + uses: actions/github-script@v7 id: find-branch with: script: | @@ -84,7 +84,7 @@ jobs: id: bump run: "tox -e update-version -- ${{ github.event.inputs.release_type }}" - name: Check milestone - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: script: | const milestones = await github.paginate(github.rest.issues.listMilestones, { @@ -168,7 +168,7 @@ jobs: run: "tox -e build-release -- --upload --no-confirm" env: TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.THE_COMPILER_PYPI_TOKEN }} + TWINE_PASSWORD: ${{ secrets.QUTEBROWSER_BOT_PYPI_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} finalize: runs-on: ubuntu-20.04 @@ -178,7 +178,7 @@ jobs: contents: write # To change release steps: - name: Publish final release - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: script: | await github.rest.repos.updateRelease({ @@ -3,7 +3,6 @@ ignore=resources.py extension-pkg-whitelist=PyQt5,PyQt6,sip load-plugins=qute_pylint.config, pylint.extensions.docstyle, - pylint.extensions.emptystring, pylint.extensions.overlapping_exceptions, pylint.extensions.code_style, pylint.extensions.comparison_placement, @@ -60,6 +59,7 @@ disable=locally-disabled, useless-param-doc, wrong-import-order, # doesn't work with qutebrowser.qt, even with known-third-party set ungrouped-imports, # ditto + use-implicit-booleaness-not-comparison-to-zero, [BASIC] function-rgx=[a-z_][a-z0-9_]{2,50}$ diff --git a/README.asciidoc b/README.asciidoc index 2b6bdfdd6..364f8fa62 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -227,7 +227,7 @@ Active https://tridactyl.xyz/[Tridactyl], https://addons.mozilla.org/en-GB/firefox/addon/vimium-ff/[Vimium-FF] * Addons for Firefox and Chrome: - https://github.com/brookhong/Surfingkeys[Surfingkeys], + https://github.com/brookhong/Surfingkeys[Surfingkeys] (https://github.com/brookhong/Surfingkeys/issues/1796[somewhat sketchy]...), https://lydell.github.io/LinkHints/[Link Hints] (hinting only), https://github.com/ueokande/vimmatic[Vimmatic] diff --git a/doc/backers.asciidoc b/doc/backers.asciidoc index bdabb5f96..81ccc14ab 100644 --- a/doc/backers.asciidoc +++ b/doc/backers.asciidoc @@ -1,6 +1,14 @@ Crowdfunding backers ==================== +2019+ +----- + +Since late 2019, qutebrowser is taking recurring donations via +https://github.com/sponsors/The-Compiler/[GitHub Sponsors] and +https://liberapay.com/The-Compiler/[Liberapay]. You can find Sponsors/Patrons +who opted to be listed as public on the respective pages. **Thank you!** + 2017 ---- diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index c5abc1aef..b7014e1f7 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -15,25 +15,36 @@ breaking changes (such as renamed commands) can happen in minor releases. // `Fixed` for any bug fixes. // `Security` to invite users to upgrade in case of vulnerabilities. -[[v3.0.1]] -v3.0.1 (unreleased) +[[v3.1.0]] +v3.1.0 (unreleased) +------------------- + +Changed +~~~~~~~ + +- (TODO) Upgraded the bundled Qt version to 6.6.1, based on Chromium 112. Note + this is only relevant for the macOS/Windows releases, on Linux those will be + upgraded via your distribution packages. + +Fixed +~~~~~ + +- Compatibility with PDF.js v4 +- Added an elaborate workaround for a bug in QtWebEngine 6.6.0 causing crashes + on Google Mail/Meet/Chat. +- Graphical glitches in Google sheets and PDF.js, again. Removed the version + restriction for the default application of + `qt.workarounds.disable_accelerated_2d_canvas` as the issue was still + evident on Qt 6.6.0. (#7489) + + +[[v3.0.2]] +v3.0.2 (2023-10-19) ------------------- Fixed ~~~~~ -- The "restore video" functionality of the `view_in_mpv` script works again on - webengine. -- Setting `url.auto_search` to `dns` works correctly now with Qt 6. -- Counts passed via keypresses now have a digit limit (4300) to avoid - exceptions due to cats sleeping on numpads. (#7834) -- Navigating via hints to a remote URL from a file:// one works again. (#7847) -- The timers related to the tab audible indicator and the auto follow timeout - no longer accumulate connections over time. (#7888) -- The workaround for crashes when using drag & drop on Wayland with Qt 6.5.2 now also - works correctly when using `wayland-egl` rather than `wayland` as Qt platform. -- Worked around a weird `TypeError` with `QProxyStyle` / `TabBarStyle` on - certain platforms with Python 3.12. - Upgraded the bundled Qt version to 6.5.3. Note this is only relevant for the macOS/Windows releases, on Linux those will be upgraded via your distribution packages. This Qt patch release comes with @@ -48,6 +59,39 @@ Fixed a critical heap buffer overflow in WebP, for which "Google is aware that an exploit [...] exists in the wild." +[[v3.0.1]] +v3.0.1 (2023-10-19) +------------------- + +Fixed +~~~~~ + +- The "restore video" functionality of the `view_in_mpv` script works again on + webengine. +- Setting `url.auto_search` to `dns` works correctly now with Qt 6. +- Counts passed via keypresses now have a digit limit (4300) to avoid + exceptions due to cats sleeping on numpads. (#7834) +- Navigating via hints to a remote URL from a file:// one works again. (#7847) +- The timers related to the tab audible indicator and the auto follow timeout + no longer accumulate connections over time. (#7888) +- The workaround for crashes when using drag & drop on Wayland with Qt 6.5.2 now also + works correctly when using `wayland-egl` rather than `wayland` as Qt platform. +- Worked around a weird `TypeError` with `QProxyStyle` / `TabBarStyle` on + certain platforms with Python 3.12. +- Removed 1px border for the downloads view, mostly noticeable when it's + transparent. +- Due to a Qt bug, cloning/undoing a tab which was not fully loaded caused + qutebrowser to crash. This is now fixed via a workaround. +- Graphical glitches in Google sheets and PDF.js via a new setting + `qt.workarounds.disable_accelerated_2d_canvas` to disable the accelerated 2D + canvas feature which defaults to enabled on affected Qt versions. (#7489) +- The download dialog should no longer freeze when browsing to directories + with many files. (#7925) +- The app.slack.com User-Agent quirk now targets chromium 112 on Qt versions + lower than 6.6.0 (previously it always targets chromium 99) (#7951) +- Workaround a Qt issue causing jpeg files to not show up in the upload file + picker when it was filtering for image filetypes (#7866) + [[v3.0.0]] v3.0.0 (2023-08-18) ------------------- diff --git a/doc/contributing.asciidoc b/doc/contributing.asciidoc index 0be2655c5..144117677 100644 --- a/doc/contributing.asciidoc +++ b/doc/contributing.asciidoc @@ -575,35 +575,46 @@ Chrome URLs ~~~~~~~~~~~ With the QtWebEngine backend, qutebrowser supports several chrome:// urls which -can be useful for debugging: +can be useful for debugging. -- chrome://accessibility/ -- chrome://appcache-internals/ -- chrome://blob-internals/ -- chrome://conversion-internals/ (QtWebEngine 5.15.3+) -- chrome://crash/ (crashes the current renderer process!) +Info pages: + +- chrome://device-log/ (QtWebEngine >= 6.3) - chrome://gpu/ -- chrome://gpuclean/ (crashes the current renderer process!) -- chrome://gpucrash/ (crashes qutebrowser!) -- chrome://gpuhang/ (hangs qutebrowser!) +- chrome://sandbox/ (Linux only) + +Misc. / Debugging pages: + +- chrome://dino/ - chrome://histograms/ +- chrome://network-errors/ +- chrome://tracing/ (QtWebEngine >= 5.15.3) +- chrome://ukm/ (QtWebEngine >= 5.15.3) +- chrome://user-actions/ (QtWebEngine >= 5.15.3) +- chrome://webrtc-logs/ (QtWebEngine >= 5.15.3) + +Internals pages: + +- chrome://accessibility/ +- chrome://appcache-internals/ (QtWebEngine < 6.4) +- chrome://attribution-internals/ (QtWebEngine >= 6.4) +- chrome://blob-internals/ +- chrome://conversion-internals/ (QtWebEngine >= 5.15.3 and < 6.4) - chrome://indexeddb-internals/ -- chrome://kill/ (kills the current renderer process!) - chrome://media-internals/ -- chrome://net-internals/ (QtWebEngine 5.15.4+) -- chrome://network-errors/ -- chrome://ppapiflashcrash/ -- chrome://ppapiflashhang/ +- chrome://net-internals/ (QtWebEngine >= 5.15.4) - chrome://process-internals/ - chrome://quota-internals/ -- chrome://sandbox/ (Linux only) - chrome://serviceworker-internals/ -- chrome://taskscheduler-internals/ (removed in QtWebEngine 5.14) -- chrome://tracing/ (QtWebEngine 5.15.3+) -- chrome://ukm/ (QtWebEngine 5.15.3+) -- chrome://user-actions/ (QtWebEngine 5.15.3+) - chrome://webrtc-internals/ -- chrome://webrtc-logs/ (QtWebEngine 5.15.3+) + +Crash/hang pages: + +- chrome://crash/ (crashes the current renderer process!) +- chrome://gpuclean/ (crashes the current renderer process!) +- chrome://gpucrash/ (crashes qutebrowser!) +- chrome://gpuhang/ (hangs qutebrowser!) +- chrome://kill/ (kills the current renderer process!) QtWebEngine internals ~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index c384ccbd6..b5c8e61a4 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -303,6 +303,7 @@ |<<qt.force_platformtheme,qt.force_platformtheme>>|Force a Qt platformtheme to use. |<<qt.force_software_rendering,qt.force_software_rendering>>|Force software rendering for QtWebEngine. |<<qt.highdpi,qt.highdpi>>|Turn on Qt HighDPI scaling. +|<<qt.workarounds.disable_accelerated_2d_canvas,qt.workarounds.disable_accelerated_2d_canvas>>|Disable accelerated 2d canvas to avoid graphical glitches. |<<qt.workarounds.locale,qt.workarounds.locale>>|Work around locale parsing issues in QtWebEngine 5.15.3. |<<qt.workarounds.remove_service_workers,qt.workarounds.remove_service_workers>>|Delete the QtWebEngine Service Worker directory on every start. |<<scrolling.bar,scrolling.bar>>|When/how to show the scrollbar. @@ -4001,6 +4002,26 @@ Type: <<types,Bool>> Default: +pass:[false]+ +[[qt.workarounds.disable_accelerated_2d_canvas]] +=== qt.workarounds.disable_accelerated_2d_canvas +Disable accelerated 2d canvas to avoid graphical glitches. +On some setups graphical issues can occur on sites like Google sheets and PDF.js. These don't occur when accelerated 2d canvas is turned off, so we do that by default. +So far these glitches only occur on some Intel graphics devices. + +This setting requires a restart. + +This setting is only available with the QtWebEngine backend. + +Type: <<types,String>> + +Valid values: + + * +always+: Disable accelerated 2d canvas + * +auto+: Disable on Qt6 < 6.6.0, enable otherwise + * +never+: Enable accelerated 2d canvas + +Default: +pass:[auto]+ + [[qt.workarounds.locale]] === qt.workarounds.locale Work around locale parsing issues in QtWebEngine 5.15.3. diff --git a/misc/org.qutebrowser.qutebrowser.appdata.xml b/misc/org.qutebrowser.qutebrowser.appdata.xml index c3447c8da..477a8194c 100644 --- a/misc/org.qutebrowser.qutebrowser.appdata.xml +++ b/misc/org.qutebrowser.qutebrowser.appdata.xml @@ -44,6 +44,8 @@ </content_rating> <releases> <!-- Add new releases here --> +<release version="3.0.2" date="2023-10-19"/> +<release version="3.0.1" date="2023-10-19"/> <release version="3.0.0" date="2023-08-18"/> <release version="2.5.4" date="2023-03-13"/> <release version="2.5.3" date="2023-02-17"/> diff --git a/misc/qutebrowser.spec b/misc/qutebrowser.spec index ecb9da68e..652f69bfb 100644 --- a/misc/qutebrowser.spec +++ b/misc/qutebrowser.spec @@ -100,6 +100,11 @@ else: DEBUG = os.environ.get('PYINSTALLER_DEBUG', '').lower() in ['1', 'true'] +if DEBUG: + options = options = [('v', None, 'OPTION')] +else: + options = [] + a = Analysis(['../qutebrowser/__main__.py'], @@ -117,6 +122,7 @@ pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) exe = EXE(pyz, a.scripts, + options, exclude_binaries=True, name='qutebrowser', icon=icon, diff --git a/misc/requirements/requirements-check-manifest.txt b/misc/requirements/requirements-check-manifest.txt index fd8e314be..91ceac76a 100644 --- a/misc/requirements/requirements-check-manifest.txt +++ b/misc/requirements/requirements-check-manifest.txt @@ -3,7 +3,7 @@ build==1.0.3 check-manifest==0.49 importlib-metadata==6.8.0 -packaging==23.1 +packaging==23.2 pyproject_hooks==1.0.0 tomli==2.0.1 -zipp==3.16.2 +zipp==3.17.0 diff --git a/misc/requirements/requirements-dev.txt b/misc/requirements/requirements-dev.txt index 3ed88d588..4d1eb9646 100644 --- a/misc/requirements/requirements-dev.txt +++ b/misc/requirements/requirements-dev.txt @@ -2,45 +2,45 @@ build==1.0.3 bump2version==1.0.1 -certifi==2023.7.22 -cffi==1.15.1 -charset-normalizer==3.2.0 -cryptography==41.0.3 +certifi==2023.11.17 +cffi==1.16.0 +charset-normalizer==3.3.2 +cryptography==41.0.5 docutils==0.20.1 github3.py==4.0.1 hunter==3.6.1 -idna==3.4 +idna==3.6 importlib-metadata==6.8.0 -importlib-resources==6.0.1 +importlib-resources==6.1.1 jaraco.classes==3.3.0 jeepney==0.8.0 -keyring==24.2.0 +keyring==24.3.0 manhole==1.8.0 markdown-it-py==3.0.0 mdurl==0.1.2 more-itertools==10.1.0 nh3==0.2.14 -packaging==23.1 +packaging==23.2 pkginfo==1.9.6 ply==3.11 pycparser==2.21 -Pygments==2.16.1 +Pygments==2.17.2 PyJWT==2.8.0 Pympler==1.0.1 pyproject_hooks==1.0.0 -PyQt-builder==1.15.2 +PyQt-builder==1.15.3 python-dateutil==2.8.2 readme-renderer==42.0 requests==2.31.0 requests-toolbelt==1.0.0 rfc3986==2.0.0 -rich==13.5.3 +rich==13.7.0 SecretStorage==3.3.3 -sip==6.7.11 +sip==6.7.12 six==1.16.0 tomli==2.0.1 twine==4.0.2 typing_extensions==4.8.0 uritemplate==4.1.1 -# urllib3==2.0.4 -zipp==3.16.2 +# urllib3==2.1.0 +zipp==3.17.0 diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt index 5df42a5dc..10d0daab5 100644 --- a/misc/requirements/requirements-flake8.txt +++ b/misc/requirements/requirements-flake8.txt @@ -2,11 +2,11 @@ attrs==23.1.0 flake8==6.1.0 -flake8-bugbear==23.9.16 -flake8-builtins==2.1.0 +flake8-bugbear==23.11.26 +flake8-builtins==2.2.0 flake8-comprehensions==3.14.0 flake8-debugger==4.1.2 -flake8-deprecated==2.1.0 +flake8-deprecated==2.2.1 flake8-docstrings==1.7.0 flake8-future-import==0.4.7 flake8-plugin-utils==1.3.3 @@ -16,7 +16,7 @@ flake8-tidy-imports==4.10.0 flake8-tuple==0.4.1 mccabe==0.7.0 pep8-naming==0.13.3 -pycodestyle==2.11.0 +pycodestyle==2.11.1 pydocstyle==6.3.0 pyflakes==3.1.0 six==1.16.0 diff --git a/misc/requirements/requirements-mypy.txt b/misc/requirements/requirements-mypy.txt index 3e0f8918e..c23c115a7 100644 --- a/misc/requirements/requirements-mypy.txt +++ b/misc/requirements/requirements-mypy.txt @@ -1,21 +1,21 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py chardet==5.2.0 -diff-cover==7.7.0 -importlib-resources==6.0.1 +diff_cover==8.0.1 +importlib-resources==6.1.1 Jinja2==3.1.2 lxml==4.9.3 MarkupSafe==2.1.3 -mypy==1.5.1 +mypy==1.7.1 mypy-extensions==1.0.0 pluggy==1.3.0 -Pygments==2.16.1 +Pygments==2.17.2 PyQt5-stubs==5.15.6.0 tomli==2.0.1 types-colorama==0.4.15.12 types-docutils==0.20.0.3 -types-Pygments==2.16.0.0 -types-PyYAML==6.0.12.11 -types-setuptools==68.2.0.0 +types-Pygments==2.17.0.0 +types-PyYAML==6.0.12.12 +types-setuptools==68.2.0.2 typing_extensions==4.8.0 -zipp==3.16.2 +zipp==3.17.0 diff --git a/misc/requirements/requirements-pyinstaller.txt b/misc/requirements/requirements-pyinstaller.txt index 2f0b35e0d..d1a2c18c9 100644 --- a/misc/requirements/requirements-pyinstaller.txt +++ b/misc/requirements/requirements-pyinstaller.txt @@ -1,5 +1,8 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -altgraph==0.17.3 -pyinstaller @ git+https://github.com/pyinstaller/pyinstaller.git@79f62ef29822169ae00cd4271390d0e3175476ad -pyinstaller-hooks-contrib==2023.8 +altgraph==0.17.4 +importlib-metadata==6.8.0 +packaging==23.2 +pyinstaller==6.2.0 +pyinstaller-hooks-contrib==2023.10 +zipp==3.17.0 diff --git a/misc/requirements/requirements-pyinstaller.txt-raw b/misc/requirements/requirements-pyinstaller.txt-raw index 7b4c8c84c..ef376ca83 100644 --- a/misc/requirements/requirements-pyinstaller.txt-raw +++ b/misc/requirements/requirements-pyinstaller.txt-raw @@ -1 +1 @@ -pyinstaller @ git+https://github.com/pyinstaller/pyinstaller.git@79f62ef29822169ae00cd4271390d0e3175476ad +pyinstaller diff --git a/misc/requirements/requirements-pylint.txt b/misc/requirements/requirements-pylint.txt index d2ed9e4a0..a782d7182 100644 --- a/misc/requirements/requirements-pylint.txt +++ b/misc/requirements/requirements-pylint.txt @@ -1,28 +1,26 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -astroid==2.15.6 -certifi==2023.7.22 -cffi==1.15.1 -charset-normalizer==3.2.0 -cryptography==41.0.3 +astroid==3.0.1 +certifi==2023.11.17 +cffi==1.16.0 +charset-normalizer==3.3.2 +cryptography==41.0.5 dill==0.3.7 github3.py==4.0.1 -idna==3.4 +idna==3.6 isort==5.12.0 -lazy-object-proxy==1.9.0 mccabe==0.7.0 pefile==2023.2.7 -platformdirs==3.10.0 +platformdirs==4.0.0 pycparser==2.21 PyJWT==2.8.0 -pylint==2.17.5 +pylint==3.0.2 python-dateutil==2.8.2 ./scripts/dev/pylint_checkers requests==2.31.0 six==1.16.0 tomli==2.0.1 -tomlkit==0.12.1 +tomlkit==0.12.3 typing_extensions==4.8.0 uritemplate==4.1.1 -# urllib3==2.0.4 -wrapt==1.15.0 +# urllib3==2.1.0 diff --git a/misc/requirements/requirements-pyqt-5.15.2.txt b/misc/requirements/requirements-pyqt-5.15.2.txt index 3feba9550..41f75871e 100644 --- a/misc/requirements/requirements-pyqt-5.15.2.txt +++ b/misc/requirements/requirements-pyqt-5.15.2.txt @@ -1,5 +1,5 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py PyQt5==5.15.2 # rq.filter: == 5.15.2 -PyQt5-sip==12.12.2 +PyQt5-sip==12.13.0 PyQtWebEngine==5.15.2 # rq.filter: == 5.15.2 diff --git a/misc/requirements/requirements-pyqt-5.15.txt b/misc/requirements/requirements-pyqt-5.15.txt index 6bb4f43fe..5f9e4828e 100644 --- a/misc/requirements/requirements-pyqt-5.15.txt +++ b/misc/requirements/requirements-pyqt-5.15.txt @@ -1,7 +1,7 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -PyQt5==5.15.9 # rq.filter: < 5.16 +PyQt5==5.15.10 # rq.filter: < 5.16 PyQt5-Qt5==5.15.2 -PyQt5-sip==12.12.2 +PyQt5-sip==12.13.0 PyQtWebEngine==5.15.6 # rq.filter: < 5.16 PyQtWebEngine-Qt5==5.15.2 diff --git a/misc/requirements/requirements-pyqt-5.txt b/misc/requirements/requirements-pyqt-5.txt index d3d62f86c..e8ee2b9c7 100644 --- a/misc/requirements/requirements-pyqt-5.txt +++ b/misc/requirements/requirements-pyqt-5.txt @@ -1,7 +1,7 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -PyQt5==5.15.9 +PyQt5==5.15.10 PyQt5-Qt5==5.15.2 -PyQt5-sip==12.12.2 +PyQt5-sip==12.13.0 PyQtWebEngine==5.15.6 PyQtWebEngine-Qt5==5.15.2 diff --git a/misc/requirements/requirements-pyqt-6.2.txt b/misc/requirements/requirements-pyqt-6.2.txt index 52bcb2ebd..e90769ddd 100644 --- a/misc/requirements/requirements-pyqt-6.2.txt +++ b/misc/requirements/requirements-pyqt-6.2.txt @@ -2,6 +2,6 @@ PyQt6==6.2.3 PyQt6-Qt6==6.2.4 -PyQt6-sip==13.5.2 +PyQt6-sip==13.6.0 PyQt6-WebEngine==6.2.1 PyQt6-WebEngine-Qt6==6.2.4 diff --git a/misc/requirements/requirements-pyqt-6.3.txt b/misc/requirements/requirements-pyqt-6.3.txt index f1b67880b..d82c623c3 100644 --- a/misc/requirements/requirements-pyqt-6.3.txt +++ b/misc/requirements/requirements-pyqt-6.3.txt @@ -2,6 +2,6 @@ PyQt6==6.3.1 PyQt6-Qt6==6.3.2 -PyQt6-sip==13.5.2 +PyQt6-sip==13.6.0 PyQt6-WebEngine==6.3.1 PyQt6-WebEngine-Qt6==6.3.2 diff --git a/misc/requirements/requirements-pyqt-6.4.txt b/misc/requirements/requirements-pyqt-6.4.txt index c9ff85771..b52e8a511 100644 --- a/misc/requirements/requirements-pyqt-6.4.txt +++ b/misc/requirements/requirements-pyqt-6.4.txt @@ -2,6 +2,6 @@ PyQt6==6.4.2 PyQt6-Qt6==6.4.3 -PyQt6-sip==13.5.2 +PyQt6-sip==13.6.0 PyQt6-WebEngine==6.4.0 PyQt6-WebEngine-Qt6==6.4.3 diff --git a/misc/requirements/requirements-pyqt-6.5.txt b/misc/requirements/requirements-pyqt-6.5.txt index cccbc20b7..5dca9ab74 100644 --- a/misc/requirements/requirements-pyqt-6.5.txt +++ b/misc/requirements/requirements-pyqt-6.5.txt @@ -1,7 +1,7 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -PyQt6==6.5.2 -PyQt6-Qt6==6.5.2 -PyQt6-sip==13.5.2 +PyQt6==6.5.3 +PyQt6-Qt6==6.5.3 +PyQt6-sip==13.6.0 PyQt6-WebEngine==6.5.0 -PyQt6-WebEngine-Qt6==6.5.2 +PyQt6-WebEngine-Qt6==6.5.3 diff --git a/misc/requirements/requirements-pyqt-6.6.txt b/misc/requirements/requirements-pyqt-6.6.txt new file mode 100644 index 000000000..0a9c72e25 --- /dev/null +++ b/misc/requirements/requirements-pyqt-6.6.txt @@ -0,0 +1,7 @@ +# This file is automatically generated by scripts/dev/recompile_requirements.py + +PyQt6==6.6.0 +PyQt6-Qt6==6.6.0 +PyQt6-sip==13.6.0 +PyQt6-WebEngine==6.6.0 +PyQt6-WebEngine-Qt6==6.6.0 diff --git a/misc/requirements/requirements-pyqt-6.6.txt-raw b/misc/requirements/requirements-pyqt-6.6.txt-raw new file mode 100644 index 000000000..7cfe6d34c --- /dev/null +++ b/misc/requirements/requirements-pyqt-6.6.txt-raw @@ -0,0 +1,4 @@ +PyQt6 >= 6.6, < 6.7 +PyQt6-Qt6 >= 6.6, < 6.7 +PyQt6-WebEngine >= 6.6, < 6.7 +PyQt6-WebEngine-Qt6 >= 6.6, < 6.7 diff --git a/misc/requirements/requirements-pyqt-6.txt b/misc/requirements/requirements-pyqt-6.txt index cccbc20b7..0a9c72e25 100644 --- a/misc/requirements/requirements-pyqt-6.txt +++ b/misc/requirements/requirements-pyqt-6.txt @@ -1,7 +1,7 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -PyQt6==6.5.2 -PyQt6-Qt6==6.5.2 -PyQt6-sip==13.5.2 -PyQt6-WebEngine==6.5.0 -PyQt6-WebEngine-Qt6==6.5.2 +PyQt6==6.6.0 +PyQt6-Qt6==6.6.0 +PyQt6-sip==13.6.0 +PyQt6-WebEngine==6.6.0 +PyQt6-WebEngine-Qt6==6.6.0 diff --git a/misc/requirements/requirements-pyqt.txt b/misc/requirements/requirements-pyqt.txt index cccbc20b7..0a9c72e25 100644 --- a/misc/requirements/requirements-pyqt.txt +++ b/misc/requirements/requirements-pyqt.txt @@ -1,7 +1,7 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -PyQt6==6.5.2 -PyQt6-Qt6==6.5.2 -PyQt6-sip==13.5.2 -PyQt6-WebEngine==6.5.0 -PyQt6-WebEngine-Qt6==6.5.2 +PyQt6==6.6.0 +PyQt6-Qt6==6.6.0 +PyQt6-sip==13.6.0 +PyQt6-WebEngine==6.6.0 +PyQt6-WebEngine-Qt6==6.6.0 diff --git a/misc/requirements/requirements-pyroma.txt b/misc/requirements/requirements-pyroma.txt index ba65ace11..2ed286912 100644 --- a/misc/requirements/requirements-pyroma.txt +++ b/misc/requirements/requirements-pyroma.txt @@ -1,17 +1,17 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py build==1.0.3 -certifi==2023.7.22 -charset-normalizer==3.2.0 +certifi==2023.11.17 +charset-normalizer==3.3.2 docutils==0.20.1 -idna==3.4 +idna==3.6 importlib-metadata==6.8.0 -packaging==23.1 -Pygments==2.16.1 +packaging==23.2 +Pygments==2.17.2 pyproject_hooks==1.0.0 pyroma==4.2 requests==2.31.0 tomli==2.0.1 -trove-classifiers==2023.8.7 -urllib3==2.0.4 -zipp==3.16.2 +trove-classifiers==2023.11.22 +urllib3==2.1.0 +zipp==3.17.0 diff --git a/misc/requirements/requirements-sphinx.txt b/misc/requirements/requirements-sphinx.txt index 77158cace..69856e27c 100644 --- a/misc/requirements/requirements-sphinx.txt +++ b/misc/requirements/requirements-sphinx.txt @@ -1,17 +1,17 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py alabaster==0.7.13 -Babel==2.12.1 -certifi==2023.7.22 -charset-normalizer==3.2.0 +Babel==2.13.1 +certifi==2023.11.17 +charset-normalizer==3.3.2 docutils==0.20.1 -idna==3.4 +idna==3.6 imagesize==1.4.1 importlib-metadata==6.8.0 Jinja2==3.1.2 MarkupSafe==2.1.3 -packaging==23.1 -Pygments==2.16.1 +packaging==23.2 +Pygments==2.17.2 pytz==2023.3.post1 requests==2.31.0 snowballstemmer==2.2.0 @@ -22,5 +22,5 @@ sphinxcontrib-htmlhelp==2.0.1 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 -urllib3==2.0.4 -zipp==3.16.2 +urllib3==2.1.0 +zipp==3.17.0 diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 116c4c1df..ec91d0003 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -2,44 +2,44 @@ attrs==23.1.0 beautifulsoup4==4.12.2 -blinker==1.6.2 -certifi==2023.7.22 -charset-normalizer==3.2.0 +blinker==1.7.0 +certifi==2023.11.17 +charset-normalizer==3.3.2 cheroot==10.0.0 click==8.1.7 -coverage==7.3.1 -exceptiongroup==1.1.3 +coverage==7.3.2 +exceptiongroup==1.2.0 execnet==2.0.2 -filelock==3.12.4 -Flask==2.3.3 +filelock==3.13.1 +Flask==3.0.0 hunter==3.6.1 -hypothesis==6.86.1 -idna==3.4 +hypothesis==6.90.0 +idna==3.6 importlib-metadata==6.8.0 iniconfig==2.0.0 itsdangerous==2.1.2 -jaraco.functools==3.9.0 +jaraco.functools==4.0.0 # Jinja2==3.1.2 -Mako==1.2.4 +Mako==1.3.0 manhole==1.8.0 # MarkupSafe==2.1.3 more-itertools==10.1.0 -packaging==23.1 -parse==1.19.1 +packaging==23.2 +parse==1.20.0 parse-type==0.6.2 pluggy==1.3.0 py-cpuinfo==9.0.0 -Pygments==2.16.1 -pytest==7.4.2 -pytest-bdd==6.1.1 +Pygments==2.17.2 +pytest==7.4.3 +pytest-bdd==7.0.0 pytest-benchmark==4.0.0 pytest-cov==4.1.0 pytest-instafail==0.5.0 -pytest-mock==3.11.1 +pytest-mock==3.12.0 pytest-qt==4.2.0 -pytest-repeat==0.9.1 -pytest-rerunfailures==12.0 -pytest-xdist==3.3.1 +pytest-repeat==0.9.3 +pytest-rerunfailures==13.0 +pytest-xdist==3.5.0 pytest-xvfb==3.0.0 PyVirtualDisplay==3.0 requests==2.31.0 @@ -47,11 +47,11 @@ requests-file==1.5.1 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.5 -tldextract==3.5.0 +tldextract==5.1.1 toml==0.10.2 tomli==2.0.1 typing_extensions==4.8.0 -urllib3==2.0.4 -vulture==2.9.1 -Werkzeug==2.3.7 -zipp==3.16.2 +urllib3==2.1.0 +vulture==2.10 +Werkzeug==3.0.1 +zipp==3.17.0 diff --git a/misc/requirements/requirements-tox.txt b/misc/requirements/requirements-tox.txt index 9be603451..e72d539ea 100644 --- a/misc/requirements/requirements-tox.txt +++ b/misc/requirements/requirements-tox.txt @@ -1,17 +1,17 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -cachetools==5.3.1 +cachetools==5.3.2 chardet==5.2.0 colorama==0.4.6 distlib==0.3.7 -filelock==3.12.4 -packaging==23.1 -pip==23.2.1 -platformdirs==3.10.0 +filelock==3.13.1 +packaging==23.2 +pip==23.3.1 +platformdirs==4.0.0 pluggy==1.3.0 pyproject-api==1.6.1 -setuptools==68.2.2 +setuptools==69.0.2 tomli==2.0.1 tox==4.11.3 -virtualenv==20.24.5 -wheel==0.41.2 +virtualenv==20.24.7 +wheel==0.42.0 diff --git a/misc/requirements/requirements-vulture.txt b/misc/requirements/requirements-vulture.txt index 1d091baf3..9bceeb7b1 100644 --- a/misc/requirements/requirements-vulture.txt +++ b/misc/requirements/requirements-vulture.txt @@ -1,4 +1,4 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py toml==0.10.2 -vulture==2.9.1 +vulture==2.10 diff --git a/misc/requirements/requirements-yamllint.txt b/misc/requirements/requirements-yamllint.txt index fd9ea256f..32589d26f 100644 --- a/misc/requirements/requirements-yamllint.txt +++ b/misc/requirements/requirements-yamllint.txt @@ -2,4 +2,4 @@ pathspec==0.11.2 PyYAML==6.0.1 -yamllint==1.32.0 +yamllint==1.33.0 diff --git a/qutebrowser/__init__.py b/qutebrowser/__init__.py index efdf91b3e..522545e12 100644 --- a/qutebrowser/__init__.py +++ b/qutebrowser/__init__.py @@ -11,7 +11,7 @@ __copyright__ = "Copyright 2014-2021 Florian Bruhin (The Compiler)" __license__ = "GPL" __maintainer__ = __author__ __email__ = "mail@qutebrowser.org" -__version__ = "3.0.0" +__version__ = "3.0.2" __version_info__ = tuple(int(part) for part in __version__.split('.')) __description__ = "A keyboard-driven, vim-like browser based on Python and Qt." diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py index 1312275dc..4d14c9cd7 100644 --- a/qutebrowser/browser/browsertab.py +++ b/qutebrowser/browser/browsertab.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Base class for a wrapper over QWebView/QWebEngineView.""" +"""Base class for a wrapper over WebView/WebEngineView.""" import enum import pathlib @@ -22,10 +22,9 @@ from qutebrowser.qt.network import QNetworkAccessManager if TYPE_CHECKING: from qutebrowser.qt.webkit import QWebHistory, QWebHistoryItem - from qutebrowser.qt.webkitwidgets import QWebPage, QWebView + from qutebrowser.qt.webkitwidgets import QWebPage from qutebrowser.qt.webenginecore import ( QWebEngineHistory, QWebEngineHistoryItem, QWebEnginePage) - from qutebrowser.qt.webenginewidgets import QWebEngineView from qutebrowser.keyinput import modeman from qutebrowser.config import config, websettings @@ -38,10 +37,12 @@ from qutebrowser.qt import sip if TYPE_CHECKING: from qutebrowser.browser import webelem from qutebrowser.browser.inspector import AbstractWebInspector + from qutebrowser.browser.webengine.webview import WebEngineView + from qutebrowser.browser.webkit.webview import WebView tab_id_gen = itertools.count(0) -_WidgetType = Union["QWebView", "QWebEngineView"] +_WidgetType = Union["WebView", "WebEngineView"] def create(win_id: int, @@ -964,7 +965,7 @@ class AbstractTabPrivate: class AbstractTab(QWidget): - """An adapter for QWebView/QWebEngineView representing a single tab.""" + """An adapter for WebView/WebEngineView representing a single tab.""" #: Signal emitted when a website requests to close this tab. window_close_requested = pyqtSignal() @@ -1058,7 +1059,7 @@ class AbstractTab(QWidget): self.before_load_started.connect(self._on_before_load_started) - def _set_widget(self, widget: Union["QWebView", "QWebEngineView"]) -> None: + def _set_widget(self, widget: _WidgetType) -> None: # pylint: disable=protected-access self._widget = widget # FIXME:v4 ignore needed for QtWebKit diff --git a/qutebrowser/browser/downloadview.py b/qutebrowser/browser/downloadview.py index 33e69284d..4b6a8b2c8 100644 --- a/qutebrowser/browser/downloadview.py +++ b/qutebrowser/browser/downloadview.py @@ -35,6 +35,7 @@ class DownloadView(QListView): QListView { background-color: {{ conf.colors.downloads.bar.bg }}; font: {{ conf.fonts.downloads }}; + border: 0; } QListView::item { diff --git a/qutebrowser/browser/pdfjs.py b/qutebrowser/browser/pdfjs.py index 467f3c605..841285deb 100644 --- a/qutebrowser/browser/pdfjs.py +++ b/qutebrowser/browser/pdfjs.py @@ -61,11 +61,11 @@ def generate_pdfjs_page(filename, url): html = html.replace('</body>', '</body><script>{}</script>'.format(script)) # WORKAROUND for the fact that PDF.js tries to use the Fetch API even with - # qute:// URLs. - pdfjs_script = '<script src="../build/pdf.js"></script>' - html = html.replace(pdfjs_script, - '<script>window.Response = undefined;</script>\n' + - pdfjs_script) + # qute:// URLs, this is probably no longer needed in PDFjs 4+. See #4235 + html = html.replace( + '<head>', + '<head>\n<script>window.Response = undefined;</script>\n' + ) return html @@ -202,10 +202,24 @@ def _read_from_system(system_path, names): return (None, None) +def get_pdfjs_js_path(): + """Checks for pdf.js main module availability and returns the path if available.""" + paths = ['build/pdf.js', 'build/pdf.mjs'] + for path in paths: + try: + get_pdfjs_res(path) + except PDFJSNotFound: + pass + else: + return path + + raise PDFJSNotFound(" or ".join(paths)) + + def is_available(): - """Return true if a pdfjs installation is available.""" + """Return true if certain parts of a pdfjs installation are available.""" try: - get_pdfjs_res('build/pdf.js') + get_pdfjs_js_path() get_pdfjs_res('web/viewer.html') except PDFJSNotFound: return False diff --git a/qutebrowser/browser/webengine/webengineinspector.py b/qutebrowser/browser/webengine/webengineinspector.py index 64ef24319..d37f41ba5 100644 --- a/qutebrowser/browser/webengine/webengineinspector.py +++ b/qutebrowser/browser/webengine/webengineinspector.py @@ -35,14 +35,19 @@ class WebEngineInspectorView(QWebEngineView): See WebEngineView.createWindow for details. """ - inspected_page = self.page().inspectedPage() + our_page = self.page() + assert our_page is not None + inspected_page = our_page.inspectedPage() + assert inspected_page is not None if machinery.IS_QT5: view = inspected_page.view() assert isinstance(view, QWebEngineView), view return view.createWindow(wintype) else: # Qt 6 newpage = inspected_page.createWindow(wintype) - return webview.WebEngineView.forPage(newpage) + ret = webview.WebEngineView.forPage(newpage) + assert ret is not None + return ret class WebEngineInspector(inspector.AbstractWebInspector): @@ -88,16 +93,17 @@ class WebEngineInspector(inspector.AbstractWebInspector): def inspect(self, page: QWebEnginePage) -> None: if not self._widget: view = WebEngineInspectorView() - inspector_page = QWebEnginePage( + new_page = QWebEnginePage( page.profile(), self ) - inspector_page.windowCloseRequested.connect(self._on_window_close_requested) - view.setPage(inspector_page) + new_page.windowCloseRequested.connect(self._on_window_close_requested) + view.setPage(new_page) self._settings = webenginesettings.WebEngineSettings(view.settings()) self._set_widget(view) inspector_page = self._widget.page() + assert inspector_page is not None assert inspector_page.profile() == page.profile() inspector_page.setInspectedPage(page) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index fb5403ae2..1275edf0b 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -24,6 +24,7 @@ from qutebrowser.browser.webengine import (spell, webenginequtescheme, cookies, webenginedownloads, notification) from qutebrowser.config import config, websettings from qutebrowser.config.websettings import AttributeInfo as Attr +from qutebrowser.misc import pakjoy from qutebrowser.utils import (standarddir, qtutils, message, log, urlmatch, usertypes, objreg, version) if TYPE_CHECKING: @@ -50,8 +51,12 @@ class _SettingsWrapper: For read operations, the default profile value is always used. """ + def _default_profile_settings(self): + assert default_profile is not None + return default_profile.settings() + def _settings(self): - yield default_profile.settings() + yield self._default_profile_settings() if private_profile: yield private_profile.settings() @@ -76,19 +81,19 @@ class _SettingsWrapper: settings.setUnknownUrlSchemePolicy(policy) def testAttribute(self, attribute): - return default_profile.settings().testAttribute(attribute) + return self._default_profile_settings().testAttribute(attribute) def fontSize(self, fonttype): - return default_profile.settings().fontSize(fonttype) + return self._default_profile_settings().fontSize(fonttype) def fontFamily(self, which): - return default_profile.settings().fontFamily(which) + return self._default_profile_settings().fontFamily(which) def defaultTextEncoding(self): - return default_profile.settings().defaultTextEncoding() + return self._default_profile_settings().defaultTextEncoding() def unknownUrlSchemePolicy(self): - return default_profile.settings().unknownUrlSchemePolicy() + return self._default_profile_settings().unknownUrlSchemePolicy() class WebEngineSettings(websettings.AbstractSettings): @@ -341,7 +346,10 @@ def _init_user_agent_str(ua): def init_user_agent(): - _init_user_agent_str(QWebEngineProfile.defaultProfile().httpUserAgent()) + """Make the default WebEngine user agent available via parsed_user_agent.""" + actual_default_profile = QWebEngineProfile.defaultProfile() + assert actual_default_profile is not None + _init_user_agent_str(actual_default_profile.httpUserAgent()) def _init_profile(profile: QWebEngineProfile) -> None: @@ -430,12 +438,21 @@ def _init_site_specific_quirks(): "AppleWebKit/{webkit_version} (KHTML, like Gecko) " "{upstream_browser_key}/{upstream_browser_version} " "Safari/{webkit_version}") - new_chrome_ua = ("Mozilla/5.0 ({os_info}) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/99 " - "Safari/537.36") firefox_ua = "Mozilla/5.0 ({os_info}; rv:90.0) Gecko/20100101 Firefox/90.0" + def maybe_newer_chrome_ua(at_least_version): + """Return a new UA if our current chrome version isn't at least at_least_version.""" + current_chome_version = version.qtwebengine_versions().chromium_major + if current_chome_version >= at_least_version: + return None + + return ( + "Mozilla/5.0 ({os_info}) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + f"Chrome/{at_least_version} " + "Safari/537.36" + ) + user_agents = [ # Needed to avoid a ""WhatsApp works with Google Chrome 36+" error # page which doesn't allow to use WhatsApp Web at all. Also see the @@ -450,13 +467,14 @@ def _init_site_specific_quirks(): # Needed because Slack adds an error which prevents using it relatively # aggressively, despite things actually working fine. - # September 2020: Qt 5.12 works, but Qt <= 5.11 shows the error. - # FIXME:qt6 Still needed? - # https://github.com/qutebrowser/qutebrowser/issues/4669 - ("ua-slack", 'https://*.slack.com/*', new_chrome_ua), + # October 2023: Slack claims they only support 112+. On #7951 at least + # one user claims it still works fine on 108 based Qt versions. + ("ua-slack", 'https://*.slack.com/*', maybe_newer_chrome_ua(112)), ] for name, pattern, ua in user_agents: + if not ua: + continue if name not in config.val.content.site_specific_quirks.skip: config.instance.set_obj('content.headers.user_agent', ua, pattern=urlmatch.UrlPattern(pattern), @@ -536,7 +554,11 @@ def init(): _global_settings = WebEngineSettings(_SettingsWrapper()) log.init.debug("Initializing profiles...") - _init_default_profile() + + # Apply potential resource patches while initializing profiles. + with pakjoy.patch_webengine(): + _init_default_profile() + init_private_profile() config.instance.changed.connect(_update_settings) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 938e100ff..1c712db5e 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -2,9 +2,10 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Wrapper over a QWebEngineView.""" +"""Wrapper over a WebEngineView.""" import math +import struct import functools import dataclasses import re @@ -12,9 +13,8 @@ import html as html_utils from typing import cast, Union, Optional from qutebrowser.qt.core import (pyqtSignal, pyqtSlot, Qt, QPoint, QPointF, QTimer, QUrl, - QObject) + QObject, QByteArray) from qutebrowser.qt.network import QAuthenticator -from qutebrowser.qt.webenginewidgets import QWebEngineView from qutebrowser.qt.webenginecore import QWebEnginePage, QWebEngineScript, QWebEngineHistory from qutebrowser.config import config @@ -612,8 +612,16 @@ class WebEngineHistoryPrivate(browsertab.AbstractHistoryPrivate): self._tab = tab self._history = cast(QWebEngineHistory, None) + def _serialize_data(self, stream_version, count, current_index): + return struct.pack(">IIi", stream_version, count, current_index) + def serialize(self): - return qtutils.serialize(self._history) + data = qtutils.serialize(self._history) + # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-117489 + if data == self._serialize_data(stream_version=4, count=1, current_index=0): + fixed = self._serialize_data(stream_version=4, count=0, current_index=-1) + return QByteArray(fixed) + return data def deserialize(self, data): qtutils.deserialize(data, self._history) @@ -1258,7 +1266,7 @@ class WebEngineTab(browsertab.AbstractTab): abort_questions = pyqtSignal() - _widget: QWebEngineView + _widget: webview.WebEngineView search: WebEngineSearch audio: WebEngineAudio printing: WebEnginePrinting diff --git a/qutebrowser/browser/webengine/webview.py b/qutebrowser/browser/webengine/webview.py index f3f652ad0..a6f2ae113 100644 --- a/qutebrowser/browser/webengine/webview.py +++ b/qutebrowser/browser/webengine/webview.py @@ -4,18 +4,22 @@ """The main browser widget for QtWebEngine.""" -from typing import List, Iterable +import mimetypes +from typing import List, Iterable, Optional from qutebrowser.qt import machinery from qutebrowser.qt.core import pyqtSignal, pyqtSlot, QUrl from qutebrowser.qt.gui import QPalette from qutebrowser.qt.webenginewidgets import QWebEngineView -from qutebrowser.qt.webenginecore import QWebEnginePage, QWebEngineCertificateError +from qutebrowser.qt.webenginecore import ( + QWebEnginePage, QWebEngineCertificateError, QWebEngineSettings, + QWebEngineHistory, +) from qutebrowser.browser import shared from qutebrowser.browser.webengine import webenginesettings, certificateerror from qutebrowser.config import config -from qutebrowser.utils import log, debug, usertypes +from qutebrowser.utils import log, debug, usertypes, qtutils _QB_FILESELECTION_MODES = { @@ -128,6 +132,57 @@ class WebEngineView(QWebEngineView): return super().contextMenuEvent(ev) + def page(self) -> "WebEnginePage": + """Return the page for this view.""" + maybe_page = super().page() + assert maybe_page is not None + assert isinstance(maybe_page, WebEnginePage) + return maybe_page + + def settings(self) -> "QWebEngineSettings": + """Return the settings for this view.""" + maybe_settings = super().settings() + assert maybe_settings is not None + return maybe_settings + + def history(self) -> "QWebEngineHistory": + """Return the history for this view.""" + maybe_history = super().history() + assert maybe_history is not None + return maybe_history + + +def extra_suffixes_workaround(upstream_mimetypes): + """Return any extra suffixes for mimetypes in upstream_mimetypes. + + Return any file extensions (aka suffixes) for mimetypes listed in + upstream_mimetypes that are not already contained in there. + + WORKAROUND: for https://bugreports.qt.io/browse/QTBUG-116905 + Affected Qt versions > 6.2.2 (probably) < 6.7.0 + """ + if not ( + qtutils.version_check("6.2.3", compiled=False) + and not qtutils.version_check("6.7.0", compiled=False) + ): + return set() + + suffixes = {entry for entry in upstream_mimetypes if entry.startswith(".")} + mimes = {entry for entry in upstream_mimetypes if "/" in entry} + python_suffixes = set() + for mime in mimes: + if mime.endswith("/*"): + python_suffixes.update( + [ + suffix + for suffix, mimetype in mimetypes.types_map.items() + if mimetype.startswith(mime[:-1]) + ] + ) + else: + python_suffixes.update(mimetypes.guess_all_extensions(mime)) + return python_suffixes - suffixes + class WebEnginePage(QWebEnginePage): @@ -261,13 +316,28 @@ class WebEnginePage(QWebEnginePage): def chooseFiles( self, mode: QWebEnginePage.FileSelectionMode, - old_files: Iterable[str], - accepted_mimetypes: Iterable[str], + old_files: Iterable[Optional[str]], + accepted_mimetypes: Iterable[Optional[str]], ) -> List[str]: """Override chooseFiles to (optionally) invoke custom file uploader.""" + accepted_mimetypes_filtered = [m for m in accepted_mimetypes if m is not None] + old_files_filtered = [f for f in old_files if f is not None] + extra_suffixes = extra_suffixes_workaround(accepted_mimetypes_filtered) + if extra_suffixes: + log.webview.debug( + "adding extra suffixes to filepicker: " + f"before={accepted_mimetypes_filtered} " + f"added={extra_suffixes}", + ) + accepted_mimetypes_filtered = list( + accepted_mimetypes_filtered + ) + list(extra_suffixes) + handler = config.val.fileselect.handler if handler == "default": - return super().chooseFiles(mode, old_files, accepted_mimetypes) + return super().chooseFiles( + mode, old_files_filtered, accepted_mimetypes_filtered, + ) assert handler == "external", handler try: qb_mode = _QB_FILESELECTION_MODES[mode] @@ -275,6 +345,8 @@ class WebEnginePage(QWebEnginePage): log.webview.warning( f"Got file selection mode {mode}, but we don't support that!" ) - return super().chooseFiles(mode, old_files, accepted_mimetypes) + return super().chooseFiles( + mode, old_files_filtered, accepted_mimetypes_filtered, + ) return shared.choose_file(qb_mode=qb_mode) diff --git a/qutebrowser/commands/command.py b/qutebrowser/commands/command.py index 5e4281474..effdcc9b0 100644 --- a/qutebrowser/commands/command.py +++ b/qutebrowser/commands/command.py @@ -62,10 +62,12 @@ class Command: COUNT_COMMAND_VALUES = [usertypes.CommandValue.count, usertypes.CommandValue.count_tab] - def __init__(self, *, handler, name, instance=None, maxsplit=None, - modes=None, not_modes=None, debug=False, deprecated=False, - no_cmd_split=False, star_args_optional=False, scope='global', - backend=None, no_replace_variables=False): + def __init__( + self, *, handler, name, instance=None, maxsplit=None, + modes=None, not_modes=None, debug=False, deprecated=False, + no_cmd_split=False, star_args_optional=False, scope='global', + backend=None, no_replace_variables=False, + ): # pylint: disable=too-many-arguments if modes is not None and not_modes is not None: raise ValueError("Only modes or not_modes can be given!") if modes is not None: diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index f0df27d2c..dd19c8579 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -385,6 +385,25 @@ qt.workarounds.locale: However, It is expected that distributions shipping QtWebEngine 5.15.3 follow up with a proper fix soon, so it is disabled by default. +qt.workarounds.disable_accelerated_2d_canvas: + type: + name: String + valid_values: + - always: Disable accelerated 2d canvas + - auto: Disable on Qt6 < 6.6.0, enable otherwise + - never: Enable accelerated 2d canvas + default: auto + backend: QtWebEngine + restart: true + desc: >- + Disable accelerated 2d canvas to avoid graphical glitches. + + On some setups graphical issues can occur on sites like Google sheets + and PDF.js. These don't occur when accelerated 2d canvas is turned off, + so we do that by default. + + So far these glitches only occur on some Intel graphics devices. + ## auto_save auto_save.interval: @@ -733,14 +752,14 @@ content.headers.user_agent: # Vim-protip: Place your cursor below this comment and run # :r!python scripts/dev/ua_fetch.py - - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 - (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" - - Chrome 114 macOS + (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36" + - Chrome 117 macOS - - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, - like Gecko) Chrome/114.0.0.0 Safari/537.36" - - Chrome 114 Win10 + like Gecko) Chrome/117.0.0.0 Safari/537.36" + - Chrome 117 Win10 - - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like - Gecko) Chrome/114.0.0.0 Safari/537.36" - - Chrome 114 Linux + Gecko) Chrome/117.0.0.0 Safari/537.36" + - Chrome 117 Linux supports_pattern: true desc: | User agent to send. diff --git a/qutebrowser/config/qtargs.py b/qutebrowser/config/qtargs.py index 7513554b3..4fa6aa43f 100644 --- a/qutebrowser/config/qtargs.py +++ b/qutebrowser/config/qtargs.py @@ -8,7 +8,7 @@ import os import sys import argparse import pathlib -from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple +from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union, Callable from qutebrowser.qt import machinery from qutebrowser.qt.core import QLocale @@ -273,10 +273,19 @@ def _qtwebengine_args( if disabled_features: yield _DISABLE_FEATURES + ','.join(disabled_features) - yield from _qtwebengine_settings_args() + yield from _qtwebengine_settings_args(versions) -_WEBENGINE_SETTINGS: Dict[str, Dict[Any, Optional[str]]] = { +_SettingValueType = Union[ + str, + Callable[ + [ + version.WebEngineVersions, + ], + str, + ], +] +_WEBENGINE_SETTINGS: Dict[str, Dict[Any, Optional[_SettingValueType]]] = { 'qt.force_software_rendering': { 'software-opengl': None, 'qt-quick': None, @@ -324,13 +333,31 @@ _WEBENGINE_SETTINGS: Dict[str, Dict[Any, Optional[str]]] = { 'auto': '--enable-experimental-web-platform-features' if machinery.IS_QT5 else None, }, + 'qt.workarounds.disable_accelerated_2d_canvas': { + 'always': '--disable-accelerated-2d-canvas', + 'never': None, + 'auto': lambda _versions: 'always' + if machinery.IS_QT6 + else 'never', + }, } -def _qtwebengine_settings_args() -> Iterator[str]: +def _qtwebengine_settings_args(versions: version.WebEngineVersions) -> Iterator[str]: for setting, args in sorted(_WEBENGINE_SETTINGS.items()): arg = args[config.instance.get(setting)] - if arg is not None: + if callable(arg): + new_value = arg(versions) + assert ( + new_value in args + ), f"qt.settings feature detection returned an unrecognized value: {new_value} for {setting}" + result = args[new_value] + if result is not None: + assert isinstance( + result, str + ), f"qt.settings feature detection returned an invalid type: {type(result)} for {setting}" + yield result + elif arg is not None: yield arg diff --git a/qutebrowser/extensions/loader.py b/qutebrowser/extensions/loader.py index 7ccdabc88..b5b232c5a 100644 --- a/qutebrowser/extensions/loader.py +++ b/qutebrowser/extensions/loader.py @@ -6,12 +6,11 @@ import pkgutil import types -import sys import pathlib import importlib import argparse import dataclasses -from typing import Callable, Iterator, List, Optional, Set, Tuple +from typing import Callable, Iterator, List, Optional, Tuple from qutebrowser.qt.core import pyqtSlot @@ -80,14 +79,6 @@ def load_components(*, skip_hooks: bool = False) -> None: def walk_components() -> Iterator[ExtensionInfo]: """Yield ExtensionInfo objects for all modules.""" - if hasattr(sys, 'frozen'): - yield from _walk_pyinstaller() - else: - yield from _walk_normal() - - -def _walk_normal() -> Iterator[ExtensionInfo]: - """Walk extensions when not using PyInstaller.""" for _finder, name, ispkg in pkgutil.walk_packages( path=components.__path__, prefix=components.__name__ + '.', @@ -102,23 +93,6 @@ def _walk_normal() -> Iterator[ExtensionInfo]: yield ExtensionInfo(name=name) -def _walk_pyinstaller() -> Iterator[ExtensionInfo]: - """Walk extensions when using PyInstaller. - - See https://github.com/pyinstaller/pyinstaller/issues/1905 - - Inspired by: - https://github.com/webcomics/dosage/blob/master/dosagelib/loader.py - """ - toc: Set[str] = set() - for importer in pkgutil.iter_importers('qutebrowser'): - if hasattr(importer, 'toc'): - toc |= importer.toc - for name in toc: - if name.startswith(components.__name__ + '.'): - yield ExtensionInfo(name=name) - - def _get_init_context() -> InitContext: """Get an InitContext object.""" return InitContext(data_dir=pathlib.Path(standarddir.data()), diff --git a/qutebrowser/keyinput/modeparsers.py b/qutebrowser/keyinput/modeparsers.py index 7db169097..05e560111 100644 --- a/qutebrowser/keyinput/modeparsers.py +++ b/qutebrowser/keyinput/modeparsers.py @@ -254,7 +254,7 @@ class RegisterKeyParser(CommandKeyParser): mode: usertypes.KeyMode, commandrunner: 'runners.CommandRunner', parent: QObject = None) -> None: - super().__init__(mode=usertypes.KeyMode.register, # type: ignore[arg-type] + super().__init__(mode=usertypes.KeyMode.register, win_id=win_id, commandrunner=commandrunner, parent=parent, diff --git a/qutebrowser/mainwindow/prompt.py b/qutebrowser/mainwindow/prompt.py index 80edf4412..84b6cd18f 100644 --- a/qutebrowser/mainwindow/prompt.py +++ b/qutebrowser/mainwindow/prompt.py @@ -15,8 +15,8 @@ from qutebrowser.qt.core import (pyqtSlot, pyqtSignal, Qt, QTimer, QDir, QModelI QItemSelectionModel, QObject, QEventLoop) from qutebrowser.qt.widgets import (QWidget, QGridLayout, QVBoxLayout, QLineEdit, QLabel, QTreeView, QSizePolicy, - QSpacerItem) -from qutebrowser.qt.gui import QFileSystemModel + QSpacerItem, QFileIconProvider) +from qutebrowser.qt.gui import (QFileSystemModel, QIcon) from qutebrowser.browser import downloads from qutebrowser.config import config, configtypes, configexc, stylesheet @@ -624,6 +624,21 @@ class LineEditPrompt(_BasePrompt): return [('prompt-accept', 'Accept'), ('mode-leave', 'Abort')] +class NullIconProvider(QFileIconProvider): + + """Returns empty icon for everything.""" + + def __init__(self): + super().__init__() + self.null_icon = QIcon() + + def icon(self, _t): + return self.null_icon + + def type(self, _info): + return 'unknown' + + class FilenamePrompt(_BasePrompt): """A prompt for a filename.""" @@ -725,6 +740,10 @@ class FilenamePrompt(_BasePrompt): def _init_fileview(self): self._file_view = QTreeView(self) self._file_model = QFileSystemModel(self) + + # avoid icon and mime type lookups, they are slow in Qt6 + self._file_model.setIconProvider(NullIconProvider()) + self._file_view.setModel(self._file_model) self._file_view.clicked.connect(self._insert_path) diff --git a/qutebrowser/misc/binparsing.py b/qutebrowser/misc/binparsing.py new file mode 100644 index 000000000..81e2e6dbb --- /dev/null +++ b/qutebrowser/misc/binparsing.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Florian Bruhin (The-Compiler) <mail@qutebrowser.org> +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Utilities for parsing binary files. + +Used by elf.py as well as pakjoy.py. +""" + +import struct +from typing import Any, IO, Tuple + + +class ParseError(Exception): + + """Raised when the file can't be parsed.""" + + +def unpack(fmt: str, fobj: IO[bytes]) -> Tuple[Any, ...]: + """Unpack the given struct format from the given file.""" + size = struct.calcsize(fmt) + data = safe_read(fobj, size) + + try: + return struct.unpack(fmt, data) + except struct.error as e: + raise ParseError(e) + + +def safe_read(fobj: IO[bytes], size: int) -> bytes: + """Read from a file, handling possible exceptions.""" + try: + return fobj.read(size) + except (OSError, OverflowError) as e: + raise ParseError(e) + + +def safe_seek(fobj: IO[bytes], pos: int) -> None: + """Seek in a file, handling possible exceptions.""" + try: + fobj.seek(pos) + except (OSError, OverflowError) as e: + raise ParseError(e) diff --git a/qutebrowser/misc/checkpyver.py b/qutebrowser/misc/checkpyver.py index d2daa41a7..596a7803a 100644 --- a/qutebrowser/misc/checkpyver.py +++ b/qutebrowser/misc/checkpyver.py @@ -15,8 +15,8 @@ try: except ImportError: # pragma: no cover try: # Python2 - from Tkinter import Tk # type: ignore[import, no-redef] - import tkMessageBox as messagebox # type: ignore[import, no-redef] # noqa: N813 + from Tkinter import Tk # type: ignore[import-not-found, no-redef] + import tkMessageBox as messagebox # type: ignore[import-not-found, no-redef] # noqa: N813 except ImportError: # Some Python without Tk Tk = None # type: ignore[misc, assignment] diff --git a/qutebrowser/misc/elf.py b/qutebrowser/misc/elf.py index aa717e790..35af5af28 100644 --- a/qutebrowser/misc/elf.py +++ b/qutebrowser/misc/elf.py @@ -44,21 +44,16 @@ This is a "best effort" parser. If it errors out, we instead end up relying on t PyQtWebEngine version, which is the next best thing. """ -import struct import enum import re import dataclasses import mmap import pathlib -from typing import Any, IO, ClassVar, Dict, Optional, Tuple, cast +from typing import IO, ClassVar, Dict, Optional, cast from qutebrowser.qt import machinery from qutebrowser.utils import log, version, qtutils - - -class ParseError(Exception): - - """Raised when the ELF file can't be parsed.""" +from qutebrowser.misc import binparsing class Bitness(enum.Enum): @@ -77,33 +72,6 @@ class Endianness(enum.Enum): big = 2 -def _unpack(fmt: str, fobj: IO[bytes]) -> Tuple[Any, ...]: - """Unpack the given struct format from the given file.""" - size = struct.calcsize(fmt) - data = _safe_read(fobj, size) - - try: - return struct.unpack(fmt, data) - except struct.error as e: - raise ParseError(e) - - -def _safe_read(fobj: IO[bytes], size: int) -> bytes: - """Read from a file, handling possible exceptions.""" - try: - return fobj.read(size) - except (OSError, OverflowError) as e: - raise ParseError(e) - - -def _safe_seek(fobj: IO[bytes], pos: int) -> None: - """Seek in a file, handling possible exceptions.""" - try: - fobj.seek(pos) - except (OSError, OverflowError) as e: - raise ParseError(e) - - @dataclasses.dataclass class Ident: @@ -125,17 +93,17 @@ class Ident: @classmethod def parse(cls, fobj: IO[bytes]) -> 'Ident': """Parse an ELF ident header from a file.""" - magic, klass, data, elfversion, osabi, abiversion = _unpack(cls._FORMAT, fobj) + magic, klass, data, elfversion, osabi, abiversion = binparsing.unpack(cls._FORMAT, fobj) try: bitness = Bitness(klass) except ValueError: - raise ParseError(f"Invalid bitness {klass}") + raise binparsing.ParseError(f"Invalid bitness {klass}") try: endianness = Endianness(data) except ValueError: - raise ParseError(f"Invalid endianness {data}") + raise binparsing.ParseError(f"Invalid endianness {data}") return cls(magic, bitness, endianness, elfversion, osabi, abiversion) @@ -172,7 +140,7 @@ class Header: def parse(cls, fobj: IO[bytes], bitness: Bitness) -> 'Header': """Parse an ELF header from a file.""" fmt = cls._FORMATS[bitness] - return cls(*_unpack(fmt, fobj)) + return cls(*binparsing.unpack(fmt, fobj)) @dataclasses.dataclass @@ -203,39 +171,39 @@ class SectionHeader: def parse(cls, fobj: IO[bytes], bitness: Bitness) -> 'SectionHeader': """Parse an ELF section header from a file.""" fmt = cls._FORMATS[bitness] - return cls(*_unpack(fmt, fobj)) + return cls(*binparsing.unpack(fmt, fobj)) def get_rodata_header(f: IO[bytes]) -> SectionHeader: """Parse an ELF file and find the .rodata section header.""" ident = Ident.parse(f) if ident.magic != b'\x7fELF': - raise ParseError(f"Invalid magic {ident.magic!r}") + raise binparsing.ParseError(f"Invalid magic {ident.magic!r}") if ident.data != Endianness.little: - raise ParseError("Big endian is unsupported") + raise binparsing.ParseError("Big endian is unsupported") if ident.version != 1: - raise ParseError(f"Only version 1 is supported, not {ident.version}") + raise binparsing.ParseError(f"Only version 1 is supported, not {ident.version}") header = Header.parse(f, bitness=ident.klass) # Read string table - _safe_seek(f, header.shoff + header.shstrndx * header.shentsize) + binparsing.safe_seek(f, header.shoff + header.shstrndx * header.shentsize) shstr = SectionHeader.parse(f, bitness=ident.klass) - _safe_seek(f, shstr.offset) - string_table = _safe_read(f, shstr.size) + binparsing.safe_seek(f, shstr.offset) + string_table = binparsing.safe_read(f, shstr.size) # Back to all sections for i in range(header.shnum): - _safe_seek(f, header.shoff + i * header.shentsize) + binparsing.safe_seek(f, header.shoff + i * header.shentsize) sh = SectionHeader.parse(f, bitness=ident.klass) name = string_table[sh.name:].split(b'\x00')[0] if name == b'.rodata': return sh - raise ParseError("No .rodata section found") + raise binparsing.ParseError("No .rodata section found") @dataclasses.dataclass @@ -262,7 +230,7 @@ def _find_versions(data: bytes) -> Versions: chromium=match.group(2).decode('ascii'), ) except UnicodeDecodeError as e: - raise ParseError(e) + raise binparsing.ParseError(e) # Here it gets even more crazy: Sometimes, we don't have the full UA in one piece # in the string table somehow (?!). However, Qt 6.2 added a separate @@ -273,20 +241,20 @@ def _find_versions(data: bytes) -> Versions: # We first get the partial Chromium version from the UA: match = re.search(pattern[:-4], data) # without trailing literal \x00 if match is None: - raise ParseError("No match in .rodata") + raise binparsing.ParseError("No match in .rodata") webengine_bytes = match.group(1) partial_chromium_bytes = match.group(2) if b"." not in partial_chromium_bytes or len(partial_chromium_bytes) < 6: # some sanity checking - raise ParseError("Inconclusive partial Chromium bytes") + raise binparsing.ParseError("Inconclusive partial Chromium bytes") # And then try to find the *full* string, stored separately, based on the # partial one we got above. pattern = br"\x00(" + re.escape(partial_chromium_bytes) + br"[0-9.]+)\x00" match = re.search(pattern, data) if match is None: - raise ParseError("No match in .rodata for full version") + raise binparsing.ParseError("No match in .rodata for full version") chromium_bytes = match.group(1) try: @@ -295,7 +263,7 @@ def _find_versions(data: bytes) -> Versions: chromium=chromium_bytes.decode('ascii'), ) except UnicodeDecodeError as e: - raise ParseError(e) + raise binparsing.ParseError(e) def _parse_from_file(f: IO[bytes]) -> Versions: @@ -316,8 +284,8 @@ def _parse_from_file(f: IO[bytes]) -> Versions: return _find_versions(cast(bytes, mmap_data)) except (OSError, OverflowError) as e: log.misc.debug(f"mmap failed ({e}), falling back to reading", exc_info=True) - _safe_seek(f, sh.offset) - data = _safe_read(f, sh.size) + binparsing.safe_seek(f, sh.offset) + data = binparsing.safe_read(f, sh.size) return _find_versions(data) @@ -344,6 +312,6 @@ def parse_webenginecore() -> Optional[Versions]: log.misc.debug(f"Got versions from ELF: {versions}") return versions - except ParseError as e: + except binparsing.ParseError as e: log.misc.debug(f"Failed to parse ELF: {e}", exc_info=True) return None diff --git a/qutebrowser/misc/pakjoy.py b/qutebrowser/misc/pakjoy.py new file mode 100644 index 000000000..a511034a2 --- /dev/null +++ b/qutebrowser/misc/pakjoy.py @@ -0,0 +1,260 @@ +# SPDX-FileCopyrightText: Florian Bruhin (The-Compiler) <mail@qutebrowser.org> +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Chromium .pak repacking. + +This entire file is a great WORKAROUND for https://bugreports.qt.io/browse/QTBUG-118157 +and the fact we can't just simply disable the hangouts extension: +https://bugreports.qt.io/browse/QTBUG-118452 + +It's yet another big hack. If you think this is bad, look at elf.py instead. + +The name of this file might or might not be inspired by a certain vegetable, +as well as the "joy" this bug has caused me. + +Useful references: + +- https://sweetscape.com/010editor/repository/files/PAK.bt (010 editor <3) +- https://textslashplain.com/2022/05/03/chromium-internals-pak-files/ +- https://github.com/myfreeer/chrome-pak-customizer +- https://source.chromium.org/chromium/chromium/src/+/main:tools/grit/pak_util.py +- https://source.chromium.org/chromium/chromium/src/+/main:tools/grit/grit/format/data_pack.py + +This is a "best effort" parser. If it errors out, we don't apply the workaround +instead of crashing. +""" + +import os +import shutil +import pathlib +import dataclasses +import contextlib +from typing import ClassVar, IO, Optional, Dict, Tuple, Iterator + +from qutebrowser.misc import binparsing, objects +from qutebrowser.utils import qtutils, standarddir, version, utils, log + +HANGOUTS_MARKER = b"// Extension ID: nkeimhogjdpnpccoofpliimaahmaaome" +HANGOUTS_ID = 36197 # as found by toofar +PAK_VERSION = 5 +RESOURCES_ENV_VAR = "QTWEBENGINE_RESOURCES_PATH" +DISABLE_ENV_VAR = "QUTE_DISABLE_PAKJOY" +CACHE_DIR_NAME = "webengine_resources_pak_quirk" +PAK_FILENAME = "qtwebengine_resources.pak" + +TARGET_URL = b"https://*.google.com/*" +REPLACEMENT_URL = b"https://qute.invalid/*" +assert len(TARGET_URL) == len(REPLACEMENT_URL) + + +@dataclasses.dataclass +class PakHeader: + + """Chromium .pak header (version 5).""" + + encoding: int # uint32 + resource_count: int # uint16 + _alias_count: int # uint16 + + _FORMAT: ClassVar[str] = "<IHH" + + @classmethod + def parse(cls, fobj: IO[bytes]) -> "PakHeader": + """Parse a PAK version 5 header from a file.""" + return cls(*binparsing.unpack(cls._FORMAT, fobj)) + + +@dataclasses.dataclass +class PakEntry: + + """Entry description in a .pak file.""" + + resource_id: int # uint16 + file_offset: int # uint32 + size: int = 0 # not in file + + _FORMAT: ClassVar[str] = "<HI" + + @classmethod + def parse(cls, fobj: IO[bytes]) -> "PakEntry": + """Parse a PAK entry from a file.""" + return cls(*binparsing.unpack(cls._FORMAT, fobj)) + + +class PakParser: + """Parse webengine pak and find patch location to disable Google Meet extension.""" + + def __init__(self, fobj: IO[bytes]) -> None: + """Parse the .pak file from the given file object.""" + pak_version = binparsing.unpack("<I", fobj)[0] + if pak_version != PAK_VERSION: + raise binparsing.ParseError(f"Unsupported .pak version {pak_version}") + + self.fobj = fobj + entries = self._read_header() + self.manifest_entry, self.manifest = self._find_manifest(entries) + + def find_patch_offset(self) -> int: + """Return byte offset of TARGET_URL into the pak file.""" + try: + return self.manifest_entry.file_offset + self.manifest.index(TARGET_URL) + except ValueError: + raise binparsing.ParseError("Couldn't find URL in manifest") + + def _maybe_get_hangouts_manifest(self, entry: PakEntry) -> Optional[bytes]: + self.fobj.seek(entry.file_offset) + data = self.fobj.read(entry.size) + + if not data.startswith(b"{") or not data.rstrip(b"\n").endswith(b"}"): + # not JSON + return None + + if HANGOUTS_MARKER not in data: + return None + + return data + + def _read_header(self) -> Dict[int, PakEntry]: + """Read the header and entry index from the .pak file.""" + entries = [] + + header = PakHeader.parse(self.fobj) + for _ in range(header.resource_count + 1): # + 1 due to sentinel at end + entries.append(PakEntry.parse(self.fobj)) + + for entry, next_entry in zip(entries, entries[1:]): + if entry.resource_id == 0: + raise binparsing.ParseError("Unexpected sentinel entry") + entry.size = next_entry.file_offset - entry.file_offset + + if entries[-1].resource_id != 0: + raise binparsing.ParseError("Missing sentinel entry") + del entries[-1] + + return {entry.resource_id: entry for entry in entries} + + def _find_manifest(self, entries: Dict[int, PakEntry]) -> Tuple[PakEntry, bytes]: + to_check = list(entries.values()) + if HANGOUTS_ID in entries: + # Most likely candidate, based on previous known ID + to_check.insert(0, entries[HANGOUTS_ID]) + + for entry in to_check: + manifest = self._maybe_get_hangouts_manifest(entry) + if manifest is not None: + return entry, manifest + + raise binparsing.ParseError("Couldn't find hangouts manifest") + + +def _find_webengine_resources() -> pathlib.Path: + """Find the QtWebEngine resources dir. + + Mirrors logic from QtWebEngine: + https://github.com/qt/qtwebengine/blob/v6.6.0/src/core/web_engine_library_info.cpp#L293-L341 + """ + if RESOURCES_ENV_VAR in os.environ: + return pathlib.Path(os.environ[RESOURCES_ENV_VAR]) + + candidates = [] + qt_data_path = qtutils.library_path(qtutils.LibraryPath.data) + if utils.is_mac: # pragma: no cover + # I'm not sure how to arrive at this path without hardcoding it + # ourselves. importlib_resources("PyQt6.Qt6") can serve as a + # replacement for the qtutils bit but it doesn't seem to help find the + # actuall Resources folder. + candidates.append( + qt_data_path / "lib" / "QtWebEngineCore.framework" / "Resources" + ) + + candidates += [ + qt_data_path / "resources", + qt_data_path, + pathlib.Path(objects.qapp.applicationDirPath()), + pathlib.Path.home() / f".{objects.qapp.applicationName()}", + ] + + for candidate in candidates: + if (candidate / PAK_FILENAME).exists(): + return candidate + + raise binparsing.ParseError("Couldn't find webengine resources dir") + + +def copy_webengine_resources() -> Optional[pathlib.Path]: + """Copy qtwebengine resources to local dir for patching.""" + resources_dir = _find_webengine_resources() + work_dir = pathlib.Path(standarddir.cache()) / CACHE_DIR_NAME + + if work_dir.exists(): + log.misc.debug(f"Removing existing {work_dir}") + shutil.rmtree(work_dir) + + versions = version.qtwebengine_versions(avoid_init=True) + if versions.webengine != utils.VersionNumber(6, 6): + # No patching needed + return None + + log.misc.debug( + "Copying webengine resources for quirk patching: " + f"{resources_dir} -> {work_dir}" + ) + + shutil.copytree(resources_dir, work_dir) + return work_dir + + +def _patch(file_to_patch: pathlib.Path) -> None: + """Apply any patches to the given pak file.""" + if not file_to_patch.exists(): + log.misc.error( + "Resource pak doesn't exist at expected location! " + f"Not applying quirks. Expected location: {file_to_patch}" + ) + return + + with open(file_to_patch, "r+b") as f: + try: + parser = PakParser(f) + log.misc.debug(f"Patching pak entry: {parser.manifest_entry}") + offset = parser.find_patch_offset() + binparsing.safe_seek(f, offset) + f.write(REPLACEMENT_URL) + except binparsing.ParseError: + log.misc.exception("Failed to apply quirk to resources pak.") + + +@contextlib.contextmanager +def patch_webengine() -> Iterator[None]: + """Apply any patches to webengine resource pak files.""" + if os.environ.get(DISABLE_ENV_VAR): + log.misc.debug(f"Not applying quirk due to {DISABLE_ENV_VAR}") + yield + return + + try: + # Still calling this on Qt != 6.6 so that the directory is cleaned up + # when not needed anymore. + webengine_resources_path = copy_webengine_resources() + except OSError: + log.misc.exception("Failed to copy webengine resources, not applying quirk") + yield + return + + if webengine_resources_path is None: + yield + return + + _patch(webengine_resources_path / PAK_FILENAME) + + old_value = os.environ.get(RESOURCES_ENV_VAR) + os.environ[RESOURCES_ENV_VAR] = str(webengine_resources_path) + + yield + + # Restore old value for subprocesses or :restart + if old_value is None: + del os.environ[RESOURCES_ENV_VAR] + else: + os.environ[RESOURCES_ENV_VAR] = old_value diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 1481ba219..b23b862a3 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -320,6 +320,7 @@ class Query: raise BugError("Cannot iterate inactive query") rec = self.query.record() fields = [rec.fieldName(i) for i in range(rec.count())] + # pylint: disable=prefer-typing-namedtuple rowtype = collections.namedtuple( # type: ignore[misc] 'ResultRow', fields) diff --git a/qutebrowser/qt/sip.py b/qutebrowser/qt/sip.py index ab5d9b907..1eb21bc27 100644 --- a/qutebrowser/qt/sip.py +++ b/qutebrowser/qt/sip.py @@ -23,7 +23,7 @@ elif machinery.USE_PYQT5: try: from PyQt5.sip import * except ImportError: - from sip import * # type: ignore[import] + from sip import * # type: ignore[import-not-found] elif machinery.USE_PYQT6: try: from PyQt6.sip import * @@ -31,6 +31,6 @@ elif machinery.USE_PYQT6: # While upstream recommends using PyQt5.sip ever since PyQt5 5.11, some # distributions still package later versions of PyQt5 with a top-level # "sip" rather than "PyQt5.sip". - from sip import * # type: ignore[import] + from sip import * # type: ignore[import-not-found] else: raise machinery.UnknownWrapper() diff --git a/qutebrowser/utils/log.py b/qutebrowser/utils/log.py index f912ebd11..3e3b407b0 100644 --- a/qutebrowser/utils/log.py +++ b/qutebrowser/utils/log.py @@ -209,6 +209,16 @@ def _init_py_warnings() -> None: message=r"Using or importing the ABCs from " r"'collections' instead of from 'collections.abc' " r"is deprecated.*") + # PyQt 5.15/6.2/6.3/6.4: + # https://riverbankcomputing.com/news/SIP_v6.7.12_Released + warnings.filterwarnings( + 'ignore', + category=DeprecationWarning, + message=( + r"sipPyTypeDict\(\) is deprecated, the extension module should use " + r"sipPyTypeDictRef\(\) instead" + ) + ) @contextlib.contextmanager diff --git a/qutebrowser/utils/qtlog.py b/qutebrowser/utils/qtlog.py index 6ec04e559..1de9181cf 100644 --- a/qutebrowser/utils/qtlog.py +++ b/qutebrowser/utils/qtlog.py @@ -10,9 +10,9 @@ import faulthandler import logging import sys import traceback -from typing import Iterator, Optional, Callable, cast +from typing import Iterator, Optional -from qutebrowser.qt import core as qtcore, machinery +from qutebrowser.qt import core as qtcore from qutebrowser.utils import log _args = None @@ -34,19 +34,6 @@ def shutdown_log() -> None: def disable_qt_msghandler() -> Iterator[None]: """Contextmanager which temporarily disables the Qt message handler.""" old_handler = qtcore.qInstallMessageHandler(None) - if machinery.IS_QT6: - # cast str to Optional[str] to be compatible with PyQt6 type hints for - # qInstallMessageHandler - old_handler = cast( - Optional[ - Callable[ - [qtcore.QtMsgType, qtcore.QMessageLogContext, Optional[str]], - None - ] - ], - old_handler, - ) - try: yield finally: diff --git a/qutebrowser/utils/qtutils.py b/qutebrowser/utils/qtutils.py index 5e36a90d2..89175ca4e 100644 --- a/qutebrowser/utils/qtutils.py +++ b/qutebrowser/utils/qtutils.py @@ -18,8 +18,8 @@ import enum import pathlib import operator import contextlib -from typing import (Any, AnyStr, TYPE_CHECKING, BinaryIO, IO, Iterator, - Optional, Union, Tuple, Protocol, cast, TypeVar) +from typing import (Any, TYPE_CHECKING, BinaryIO, IO, Iterator, Literal, + Optional, Union, Tuple, Protocol, cast, overload, TypeVar) from qutebrowser.qt import machinery, sip from qutebrowser.qt.core import (qVersion, QEventLoop, QDataStream, QByteArray, @@ -80,10 +80,25 @@ def version_check(version: str, compiled: bool = True) -> bool: """Check if the Qt runtime version is the version supplied or newer. + By default this function will check `version` against: + + 1. the runtime Qt version (from qVersion()) + 2. the Qt version that PyQt was compiled against (from QT_VERSION_STR) + 3. the PyQt version (from PYQT_VERSION_STR) + + With `compiled=False` only the runtime Qt version (1) is checked. + + You can often run older PyQt versions against newer Qt versions, but you + won't be able to access any APIs that were only added in the newer Qt + version. So if you want to check if a new feature is supported, use the + default behavior. If you just want to check the underlying Qt version, + pass `compiled=False`. + Args: version: The version to check against. exact: if given, check with == instead of >= - compiled: Set to False to not check the compiled version. + compiled: Set to False to not check the compiled Qt version or the + PyQt version. """ if compiled and exact: raise ValueError("Can't use compiled=True with exact=True!") @@ -221,12 +236,32 @@ def deserialize_stream(stream: QDataStream, obj: _QtSerializableType) -> None: check_qdatastream(stream) +@overload +@contextlib.contextmanager +def savefile_open( + filename: str, + binary: Literal[False] = ..., + encoding: str = 'utf-8' +) -> Iterator[IO[str]]: + ... + + +@overload +@contextlib.contextmanager +def savefile_open( + filename: str, + binary: Literal[True], + encoding: str = 'utf-8' +) -> Iterator[IO[bytes]]: + ... + + @contextlib.contextmanager def savefile_open( filename: str, binary: bool = False, encoding: str = 'utf-8' -) -> Iterator[IO[AnyStr]]: +) -> Iterator[Union[IO[str], IO[bytes]]]: """Context manager to easily use a QSaveFile.""" f = QSaveFile(filename) cancelled = False @@ -238,7 +273,7 @@ def savefile_open( dev = cast(BinaryIO, PyQIODevice(f)) if binary: - new_f: IO[Any] = dev # FIXME:mypy Why doesn't AnyStr work? + new_f: Union[IO[str], IO[bytes]] = dev else: new_f = io.TextIOWrapper(dev, encoding=encoding) diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py index 75df73ffa..59da5b5f0 100644 --- a/qutebrowser/utils/version.py +++ b/qutebrowser/utils/version.py @@ -480,7 +480,7 @@ def _pdfjs_version() -> str: A string with the version number. """ try: - pdfjs_file, file_path = pdfjs.get_pdfjs_res_and_path('build/pdf.js') + pdfjs_file, file_path = pdfjs.get_pdfjs_res_and_path(pdfjs.get_pdfjs_js_path()) except pdfjs.PDFJSNotFound: return 'no' else: @@ -686,7 +686,7 @@ class WebEngineVersions: return cls._CHROMIUM_VERSIONS.get(minor_version) @classmethod - def from_api(cls, qtwe_version: str, chromium_version: str) -> 'WebEngineVersions': + def from_api(cls, qtwe_version: str, chromium_version: Optional[str]) -> 'WebEngineVersions': """Get the versions based on the exact versions. This is called if we have proper APIs to get the versions easily @@ -796,8 +796,10 @@ def qtwebengine_versions(*, avoid_init: bool = False) -> WebEngineVersions: except ImportError: pass # Needs QtWebEngine 6.2+ with PyQtWebEngine 6.3.1+ else: + qtwe_version = qWebEngineVersion() + assert qtwe_version is not None return WebEngineVersions.from_api( - qtwe_version=qWebEngineVersion(), + qtwe_version=qtwe_version, chromium_version=qWebEngineChromiumVersion(), ) diff --git a/requirements.txt b/requirements.txt index 01f6236c7..81e0d0606 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,12 +2,12 @@ adblock==0.6.0 colorama==0.4.6 -importlib-resources==6.0.1 ; python_version=="3.8.*" +importlib-resources==6.1.1 ; python_version=="3.8.*" Jinja2==3.1.2 MarkupSafe==2.1.3 -Pygments==2.16.1 +Pygments==2.17.2 PyYAML==6.0.1 -zipp==3.16.2 +zipp==3.17.0 # Unpinned due to recompile_requirements.py limitations pyobjc-core ; sys_platform=="darwin" pyobjc-framework-Cocoa ; sys_platform=="darwin" diff --git a/scripts/dev/changelog_urls.json b/scripts/dev/changelog_urls.json index 5a9c9b34a..cb1c7d1bb 100644 --- a/scripts/dev/changelog_urls.json +++ b/scripts/dev/changelog_urls.json @@ -1,9 +1,8 @@ { - "pylint": "https://pylint.pycqa.org/en/latest/whatsnew/2/index.html", + "pylint": "https://pylint.pycqa.org/en/latest/whatsnew/3/index.html", "tomlkit": "https://github.com/sdispater/tomlkit/blob/master/CHANGELOG.md", "dill": "https://github.com/uqfoundation/dill/commits/master", "isort": "https://github.com/PyCQA/isort/blob/main/CHANGELOG.md", - "lazy-object-proxy": "https://github.com/ionelmc/python-lazy-object-proxy/blob/master/CHANGELOG.rst", "mccabe": "https://github.com/PyCQA/mccabe#changes", "pytest-cov": "https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst", "pytest-xdist": "https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst", @@ -59,7 +58,7 @@ "pep8-naming": "https://github.com/PyCQA/pep8-naming/blob/main/CHANGELOG.rst", "pycodestyle": "https://github.com/PyCQA/pycodestyle/blob/main/CHANGES.txt", "pyflakes": "https://github.com/PyCQA/pyflakes/blob/master/NEWS.rst", - "cffi": "https://foss.heptapod.net/pypy/cffi/-/blob/branch/default/doc/source/whatsnew.rst", + "cffi": "https://github.com/python-cffi/cffi/blob/main/doc/source/whatsnew.rst", "astroid": "https://github.com/PyCQA/astroid/blob/main/ChangeLog", "pytest-instafail": "https://github.com/pytest-dev/pytest-instafail/blob/master/CHANGES.rst", "coverage": "https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst", @@ -93,7 +92,6 @@ "altgraph": "https://github.com/ronaldoussoren/altgraph/blob/master/doc/changelog.rst", "urllib3": "https://github.com/urllib3/urllib3/blob/main/CHANGES.rst", "lxml": "https://github.com/lxml/lxml/blob/master/CHANGES.txt", - "wrapt": "https://github.com/GrahamDumpleton/wrapt/blob/develop/docs/changes.rst", "cryptography": "https://cryptography.io/en/latest/changelog.html", "toml": "https://github.com/uiri/toml/releases", "tomli": "https://github.com/hukkin/tomli/blob/master/CHANGELOG.md", @@ -121,7 +119,7 @@ "idna": "https://github.com/kjd/idna/blob/master/HISTORY.rst", "tldextract": "https://github.com/john-kurkowski/tldextract/blob/master/CHANGELOG.md", "typing_extensions": "https://github.com/python/typing_extensions/blob/main/CHANGELOG.md", - "diff-cover": "https://github.com/Bachmann1234/diff_cover/blob/main/CHANGELOG", + "diff_cover": "https://github.com/Bachmann1234/diff_cover/blob/main/CHANGELOG", "beautifulsoup4": "https://bazaar.launchpad.net/~leonardr/beautifulsoup/bs4/view/head:/CHANGELOG", "check-manifest": "https://github.com/mgedmin/check-manifest/blob/master/CHANGES.rst", "yamllint": "https://github.com/adrienverge/yamllint/blob/master/CHANGELOG.rst", diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index a3a9bf644..38a8f6ca1 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -123,6 +123,8 @@ PERFECT_FILES = [ 'qutebrowser/misc/objects.py'), ('tests/unit/misc/test_throttle.py', 'qutebrowser/misc/throttle.py'), + ('tests/unit/misc/test_pakjoy.py', + 'qutebrowser/misc/pakjoy.py'), (None, 'qutebrowser/mainwindow/statusbar/keystring.py'), @@ -328,10 +330,6 @@ def main_check(): print("or check https://codecov.io/github/qutebrowser/qutebrowser") print() - if scriptutils.ON_CI: - print("Keeping coverage.xml on CI.") - else: - os.remove('coverage.xml') return 1 if messages else 0 @@ -352,7 +350,6 @@ def main_check_all(): '--cov-report', 'xml', test_file], check=True) with open('coverage.xml', encoding='utf-8') as f: messages = check(f, [(test_file, src_file)]) - os.remove('coverage.xml') messages = [msg for msg in messages if msg.typ == MsgType.insufficient_coverage] diff --git a/scripts/dev/ci/docker/Dockerfile.j2 b/scripts/dev/ci/docker/Dockerfile.j2 index 3a1adbdef..4b958babd 100644 --- a/scripts/dev/ci/docker/Dockerfile.j2 +++ b/scripts/dev/ci/docker/Dockerfile.j2 @@ -41,7 +41,13 @@ RUN pacman -U --noconfirm \ https://archive.archlinux.org/packages/p/python/python-3.10.10-1-x86_64.pkg.tar.zst \ https://archive.archlinux.org/packages/i/icu/icu-72.1-2-x86_64.pkg.tar.zst \ https://archive.archlinux.org/packages/l/libxml2/libxml2-2.10.4-4-x86_64.pkg.tar.zst \ - https://archive.archlinux.org/packages/q/qt5-base/qt5-base-5.15.10%2Bkde%2Br129-3-x86_64.pkg.tar.zst + https://archive.archlinux.org/packages/q/qt5-base/qt5-base-5.15.10%2Bkde%2Br129-3-x86_64.pkg.tar.zst \ + https://archive.archlinux.org/packages/q/qt5-declarative/qt5-declarative-5.15.10%2Bkde%2Br31-1-x86_64.pkg.tar.zst \ + https://archive.archlinux.org/packages/q/qt5-translations/qt5-translations-5.15.10-1-any.pkg.tar.zst \ + https://archive.archlinux.org/packages/q/qt5-sensors/qt5-sensors-5.15.10-1-x86_64.pkg.tar.zst \ + https://archive.archlinux.org/packages/q/qt5-location/qt5-location-5.15.10%2Bkde%2Br5-1-x86_64.pkg.tar.zst \ + https://archive.archlinux.org/packages/q/qt5-webchannel/qt5-webchannel-5.15.10%2Bkde%2Br3-1-x86_64.pkg.tar.zst + RUN python3 -m ensurepip RUN python3 -m pip install tox pyqt5-sip {% endif %} @@ -65,4 +71,4 @@ WORKDIR /home/user CMD git clone /outside qutebrowser.git && \ cd qutebrowser.git && \ - tox -e {% if qt6 %}py-qt6{% else %}py{% endif %} + tox -e {% if qt6 %}py-qt6{% else %}py-qt5{% endif %} diff --git a/scripts/dev/ci/problemmatchers.py b/scripts/dev/ci/problemmatchers.py index fa623dec7..3316c5597 100644 --- a/scripts/dev/ci/problemmatchers.py +++ b/scripts/dev/ci/problemmatchers.py @@ -160,13 +160,17 @@ MATCHERS = { "tests": [ { # pytest test summary output + # Examples (with ANSI color codes around FAILED|ERROR and the + # function name): + # FAILED tests/end2end/features/test_keyinput_bdd.py::test_fakekey_sending_special_key_to_the_website - end2end.fixtures.testprocess.WaitForTimeout: Timed out after 15000ms waiting for {'category': 'js', 'message': '[*] key press: 27'}. + # ERROR tests/end2end/test_insert_mode.py::test_insert_mode[100-textarea.html-qute-textarea-clipboard-qutebrowser] - Failed: Logged unexpected errors: "severity": "error", "pattern": [ { - "regexp": r'^=+ short test summary info =+$', + "regexp": r'^.*=== short test summary info ===.*$', }, { - "regexp": r"^((ERROR|FAILED) .*)", + "regexp": r"^[^ ]*((ERROR|FAILED)[^ ]* .*)$", "message": 1, "loop": True, } diff --git a/scripts/dev/pylint_checkers/qute_pylint/config.py b/scripts/dev/pylint_checkers/qute_pylint/config.py index 283de5d35..be5bae082 100644 --- a/scripts/dev/pylint_checkers/qute_pylint/config.py +++ b/scripts/dev/pylint_checkers/qute_pylint/config.py @@ -21,7 +21,6 @@ class ConfigChecker(checkers.BaseChecker): """Custom astroid checker for config calls.""" - __implements__ = interfaces.IAstroidChecker name = 'config' msgs = { 'E9998': ('%s is no valid config option.', # flake8: disable=S001 @@ -31,7 +30,7 @@ class ConfigChecker(checkers.BaseChecker): priority = -1 printed_warning = False - @utils.check_messages('bad-config-option') + @utils.only_required_for_messages('bad-config-option') def visit_attribute(self, node): """Visit a getattr node.""" # We're only interested in the end of a config.val.foo.bar chain diff --git a/scripts/link_pyqt.py b/scripts/link_pyqt.py index c188660a6..108696317 100644 --- a/scripts/link_pyqt.py +++ b/scripts/link_pyqt.py @@ -107,8 +107,11 @@ def get_lib_path(executable, name, required=True): return data elif prefix == 'ImportError': if required: - raise Error("Could not import {} with {}: {}!".format( - name, executable, data)) + wrapper = os.environ["QUTE_QT_WRAPPER"] + raise Error( + f"Could not import {name} with {executable}: {data} " + f"(QUTE_QT_WRAPPER: {wrapper})" + ) return None else: raise ValueError("Unexpected output: {!r}".format(output)) diff --git a/tests/conftest.py b/tests/conftest.py index 2fea48c43..9d7c5c29c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -197,7 +197,12 @@ def qapp_args(): """Make QtWebEngine unit tests run on older Qt versions + newer kernels.""" if testutils.disable_seccomp_bpf_sandbox(): return [sys.argv[0], testutils.DISABLE_SECCOMP_BPF_FLAG] - return [sys.argv[0]] + + # Disabling PaintHoldingCrossOrigin makes tests needing UI interaction with + # QtWebEngine more reliable. + # Only needed with QtWebEngine and Qt 6.5, but Qt just ignores arguments it + # doesn't know about anyways. + return [sys.argv[0], "--webEngineArgs", "--disable-features=PaintHoldingCrossOrigin"] @pytest.fixture(scope='session') diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index 9a093bba8..de9b490ca 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -66,7 +66,8 @@ def is_ignored_lowlevel_message(message): ( 'libva error: vaGetDriverNameByIndex() failed with unknown libva error, ' 'driver_name = (null)' - ) + ), + 'libva error: vaGetDriverNames() failed with unknown libva error', ] return any(testutils.pattern_match(pattern=pattern, value=message) for pattern in ignored_messages) @@ -389,7 +390,8 @@ class QuteProc(testprocess.Process): '--json-logging', '--loglevel', 'vdebug', '--backend', backend, '--debug-flag', 'no-sql-history', '--debug-flag', 'werror', '--debug-flag', - 'test-notification-service'] + 'test-notification-service', + '--qt-flag', 'disable-features=PaintHoldingCrossOrigin'] if self.request.config.webengine and testutils.disable_seccomp_bpf_sandbox(): args += testutils.DISABLE_SECCOMP_BPF_ARGS diff --git a/tests/end2end/test_invocations.py b/tests/end2end/test_invocations.py index af81781f6..a55efb129 100644 --- a/tests/end2end/test_invocations.py +++ b/tests/end2end/test_invocations.py @@ -15,6 +15,7 @@ import re import json import platform from contextlib import nullcontext as does_not_raise +from unittest.mock import ANY import pytest from qutebrowser.qt.core import QProcess, QPoint @@ -882,30 +883,86 @@ def test_sandboxing( line.expected = True pytest.skip("chrome://sandbox/ not supported") + if len(text.split("\n")) == 1: + # Try again, maybe the JS hasn't run yet? + text = quteproc_new.get_content() + print(text) + bpf_text = "Seccomp-BPF sandbox" yama_text = "Ptrace Protection with Yama LSM" - header, *lines, empty, result = text.split("\n") - assert not empty - - expected_status = { - "Layer 1 Sandbox": "Namespace" if has_namespaces else "None", + if not utils.is_windows: + header, *lines, empty, result = text.split("\n") + assert not empty - "PID namespaces": "Yes" if has_namespaces else "No", - "Network namespaces": "Yes" if has_namespaces else "No", + expected_status = { + "Layer 1 Sandbox": "Namespace" if has_namespaces else "None", - bpf_text: "Yes" if has_seccomp else "No", - f"{bpf_text} supports TSYNC": "Yes" if has_seccomp else "No", + "PID namespaces": "Yes" if has_namespaces else "No", + "Network namespaces": "Yes" if has_namespaces else "No", - f"{yama_text} (Broker)": "Yes" if has_yama else "No", - f"{yama_text} (Non-broker)": "Yes" if has_yama_non_broker else "No", - } + bpf_text: "Yes" if has_seccomp else "No", + f"{bpf_text} supports TSYNC": "Yes" if has_seccomp else "No", - assert header == "Sandbox Status" - assert result == expected_result + f"{yama_text} (Broker)": "Yes" if has_yama else "No", + f"{yama_text} (Non-broker)": "Yes" if has_yama_non_broker else "No", + } - status = dict(line.split("\t") for line in lines) - assert status == expected_status + assert header == "Sandbox Status" + assert result == expected_result + + status = dict(line.split("\t") for line in lines) + assert status == expected_status + + else: # utils.is_windows + # The sandbox page on Windows if different that Linux and macOS. It's + # a lot more complex. There is a table up top with lots of columns and + # a row per tab and helper process then a json object per row down + # below with even more detail (which we ignore). + # https://www.chromium.org/Home/chromium-security/articles/chrome-sandbox-diagnostics-for-windows/ + + # We're not getting full coverage of the table and there doesn't seem + # to be a simple summary like for linux. The "Sandbox" and "Lockdown" + # column are probably the key ones. + # We are looking at all the rows in the table for the sake of + # completeness, but I expect there will always be just one row with a + # renderer process in it for this test. If other helper processes pop + # up we might want to exclude them. + lines = text.split("\n") + assert lines.pop(0) == "Sandbox Status" + header = lines.pop(0).split("\t") + rows = [] + current_line = lines.pop(0) + while current_line.strip(): + if lines[0].startswith("\t"): + # Continuation line. Not sure how to 100% identify them + # but new rows should start with a process ID. + current_line += lines.pop(0) + continue + + columns = current_line.split("\t") + assert len(header) == len(columns) + rows.append(dict(zip(header, columns))) + current_line = lines.pop(0) + + assert rows + + # I'm using has_namespaces as a proxy for "should be sandboxed" here, + # which is a bit lazy but its either that or match on the text + # "sandboxing" arg. The seccomp-bpf arg does nothing on windows, so + # we only have the off and on states. + for row in rows: + assert row == { + "Process": ANY, + "Type": "Renderer", + "Name": "", + "Sandbox": "Renderer" if has_namespaces else "Not Sandboxed", + "Lockdown": "Lockdown" if has_namespaces else "", + "Integrity": ANY if has_namespaces else "", + "Mitigations": ANY if has_namespaces else "", + "Component Filter": ANY if has_namespaces else "", + "Lowbox/AppContainer": "", + } @pytest.mark.not_frozen diff --git a/tests/unit/browser/test_pdfjs.py b/tests/unit/browser/test_pdfjs.py index fe2fea9a0..cb5c26229 100644 --- a/tests/unit/browser/test_pdfjs.py +++ b/tests/unit/browser/test_pdfjs.py @@ -193,6 +193,33 @@ def test_is_available(available, mocker): assert pdfjs.is_available() == available +@pytest.mark.parametrize('found_file', [ + "build/pdf.js", + "build/pdf.mjs", +]) +def test_get_pdfjs_js_path(found_file: str, monkeypatch: pytest.MonkeyPatch): + def fake_pdfjs_res(requested): + if requested.endswith(found_file): + return + raise pdfjs.PDFJSNotFound(requested) + + monkeypatch.setattr(pdfjs, 'get_pdfjs_res', fake_pdfjs_res) + assert pdfjs.get_pdfjs_js_path() == found_file + + +def test_get_pdfjs_js_path_none(monkeypatch: pytest.MonkeyPatch): + def fake_pdfjs_res(requested): + raise pdfjs.PDFJSNotFound(requested) + + monkeypatch.setattr(pdfjs, 'get_pdfjs_res', fake_pdfjs_res) + + with pytest.raises( + pdfjs.PDFJSNotFound, + match="Path 'build/pdf.js or build/pdf.mjs' not found" + ): + pdfjs.get_pdfjs_js_path() + + @pytest.mark.parametrize('mimetype, url, enabled, expected', [ # PDF files ('application/pdf', 'http://www.example.com', True, True), diff --git a/tests/unit/browser/webengine/test_webview.py b/tests/unit/browser/webengine/test_webview.py index 98bf34f3b..f14a896b6 100644 --- a/tests/unit/browser/webengine/test_webview.py +++ b/tests/unit/browser/webengine/test_webview.py @@ -4,11 +4,13 @@ import re import dataclasses +import mimetypes import pytest webview = pytest.importorskip('qutebrowser.browser.webengine.webview') from qutebrowser.qt.webenginecore import QWebEnginePage +from qutebrowser.utils import qtutils from helpers import testutils @@ -58,3 +60,82 @@ def test_enum_mappings(enum_type, naming, mapping): for name, val in members: mapped = mapping[val] assert camel_to_snake(naming, name) == mapped.name + + +@pytest.fixture +def suffix_mocks(monkeypatch): + types_map = { + ".jpg": "image/jpeg", + ".jpe": "image/jpeg", + ".png": "image/png", + ".m4v": "video/mp4", + ".mpg4": "video/mp4", + } + mimetypes_map = {} # mimetype -> [suffixes] map + for suffix, mime in types_map.items(): + mimetypes_map[mime] = mimetypes_map.get(mime, []) + [suffix] + + def guess(mime): + return mimetypes_map.get(mime, []) + + monkeypatch.setattr(mimetypes, "guess_all_extensions", guess) + monkeypatch.setattr(mimetypes, "types_map", types_map) + + def version(string, compiled=True): + assert compiled is False + if string == "6.2.3": + return True + if string == "6.7.0": + return False + raise AssertionError(f"unexpected version {string}") + + monkeypatch.setattr(qtutils, "version_check", version) + + +EXTRA_SUFFIXES_PARAMS = [ + (["image/jpeg"], {".jpg", ".jpe"}), + (["image/jpeg", ".jpeg"], {".jpg", ".jpe"}), + (["image/jpeg", ".jpg", ".jpe"], set()), + ( + [ + ".jpg", + ], + set(), + ), # not sure why black reformats this one and not the others + (["image/jpeg", "video/mp4"], {".jpg", ".jpe", ".m4v", ".mpg4"}), + (["image/*"], {".jpg", ".jpe", ".png"}), + (["image/*", ".jpg"], {".jpe", ".png"}), +] + + +@pytest.mark.parametrize("before, extra", EXTRA_SUFFIXES_PARAMS) +def test_suffixes_workaround_extras_returned(suffix_mocks, before, extra): + assert extra == webview.extra_suffixes_workaround(before) + + +@pytest.mark.parametrize("before, extra", EXTRA_SUFFIXES_PARAMS) +def test_suffixes_workaround_choosefiles_args( + mocker, + suffix_mocks, + config_stub, + before, + extra, +): + # mock super() to avoid calling into the base class' chooseFiles() + # implementation. + mocked_super = mocker.patch("qutebrowser.browser.webengine.webview.super") + + # We can pass None as "self" because we aren't actually using anything from + # "self" for this test. That saves us having to initialize the class and + # mock all the stuff required for __init__() + webview.WebEnginePage.chooseFiles( + None, + QWebEnginePage.FileSelectionMode.FileSelectOpen, + [], + before, + ) + expected = set(before).union(extra) + + assert len(mocked_super().chooseFiles.call_args_list) == 1 + called_with = mocked_super().chooseFiles.call_args_list[0][0][2] + assert sorted(called_with) == sorted(expected) diff --git a/tests/unit/config/test_qtargs.py b/tests/unit/config/test_qtargs.py index 1cb149430..2414d4ba9 100644 --- a/tests/unit/config/test_qtargs.py +++ b/tests/unit/config/test_qtargs.py @@ -51,6 +51,7 @@ def reduce_args(config_stub, version_patcher, monkeypatch): config_stub.val.content.headers.referer = 'always' config_stub.val.scrolling.bar = 'never' config_stub.val.qt.chromium.experimental_web_platform_features = 'never' + config_stub.val.qt.workarounds.disable_accelerated_2d_canvas = 'never' monkeypatch.setattr(qtargs.utils, 'is_mac', False) # Avoid WebRTC pipewire feature monkeypatch.setattr(qtargs.utils, 'is_linux', False) @@ -154,6 +155,32 @@ class TestWebEngineArgs: assert '--disable-in-process-stack-traces' in args assert '--enable-in-process-stack-traces' not in args + @pytest.mark.parametrize( + 'qt6, value, has_arg', + [ + (False, 'auto', False), + (True, 'auto', True), + (True, 'always', True), + (True, 'never', False), + ], + ) + def test_accelerated_2d_canvas( + self, + parser, + version_patcher, + config_stub, + monkeypatch, + qt6, + value, + has_arg, + ): + config_stub.val.qt.workarounds.disable_accelerated_2d_canvas = value + monkeypatch.setattr(machinery, 'IS_QT6', qt6) + + parsed = parser.parse_args([]) + args = qtargs.qt_args(parsed) + assert ('--disable-accelerated-2d-canvas' in args) == has_arg + @pytest.mark.parametrize('flags, args', [ ([], []), (['--debug-flag', 'chromium'], ['--enable-logging', '--v=1']), diff --git a/tests/unit/extensions/test_loader.py b/tests/unit/extensions/test_loader.py index fd15130ba..a2a99f305 100644 --- a/tests/unit/extensions/test_loader.py +++ b/tests/unit/extensions/test_loader.py @@ -20,16 +20,10 @@ def test_on_walk_error(): def test_walk_normal(): - names = [info.name for info in loader._walk_normal()] + names = [info.name for info in loader.walk_components()] assert 'qutebrowser.components.scrollcommands' in names -def test_walk_pyinstaller(): - # We can't test whether we get something back without being frozen by - # PyInstaller, but at least we can test that we don't crash. - list(loader._walk_pyinstaller()) - - def test_load_component(monkeypatch): monkeypatch.setattr(objects, 'commands', {}) diff --git a/tests/unit/misc/test_elf.py b/tests/unit/misc/test_elf.py index 88b984dcd..6ae23357c 100644 --- a/tests/unit/misc/test_elf.py +++ b/tests/unit/misc/test_elf.py @@ -9,7 +9,7 @@ import pytest import hypothesis from hypothesis import strategies as hst -from qutebrowser.misc import elf +from qutebrowser.misc import elf, binparsing from qutebrowser.utils import utils @@ -117,7 +117,7 @@ def test_find_versions(data, expected): ), ]) def test_find_versions_invalid(data, message): - with pytest.raises(elf.ParseError) as excinfo: + with pytest.raises(binparsing.ParseError) as excinfo: elf._find_versions(data) assert str(excinfo.value) == message @@ -132,5 +132,5 @@ def test_hypothesis(data): fobj = io.BytesIO(data) try: elf._parse_from_file(fobj) - except elf.ParseError as e: + except binparsing.ParseError as e: print(e) diff --git a/tests/unit/misc/test_pakjoy.py b/tests/unit/misc/test_pakjoy.py new file mode 100644 index 000000000..65d02ec7e --- /dev/null +++ b/tests/unit/misc/test_pakjoy.py @@ -0,0 +1,441 @@ +# SPDX-FileCopyrightText: Florian Bruhin (The-Compiler) <mail@qutebrowser.org> +# +# SPDX-License-Identifier: GPL-3.0-or-later + +import os +import io +import json +import struct +import pathlib +import logging +import shutil + +import pytest + +from qutebrowser.misc import pakjoy, binparsing +from qutebrowser.utils import utils, version, standarddir + + +pytest.importorskip("qutebrowser.qt.webenginecore") + + +pytestmark = pytest.mark.usefixtures("cache_tmpdir") + + +versions = version.qtwebengine_versions(avoid_init=True) + + +# Used to skip happy path tests with the real resources file. +# +# Since we don't know how reliably the Google Meet hangouts extensions is +# reliably in the resource files, and this quirk is only targeting 6.6 +# anyway. +skip_if_unsupported = pytest.mark.skipif( + versions.webengine != utils.VersionNumber(6, 6), + reason="Code under test only runs on 6.6", +) + + +@pytest.fixture(autouse=True) +def prepare_env(qapp, monkeypatch): + monkeypatch.setattr(pakjoy.objects, "qapp", qapp) + monkeypatch.delenv(pakjoy.RESOURCES_ENV_VAR, raising=False) + monkeypatch.delenv(pakjoy.DISABLE_ENV_VAR, raising=False) + + +def patch_version(monkeypatch, *args): + monkeypatch.setattr( + pakjoy.version, + "qtwebengine_versions", + lambda **kwargs: version.WebEngineVersions( + webengine=utils.VersionNumber(*args), + chromium=None, + source="unittest", + ), + ) + + +@pytest.fixture +def unaffected_version(monkeypatch): + patch_version(monkeypatch, 6, 6, 1) + + +@pytest.fixture +def affected_version(monkeypatch): + patch_version(monkeypatch, 6, 6) + + +@pytest.mark.parametrize("workdir_exists", [True, False]) +def test_version_gate(cache_tmpdir, unaffected_version, mocker, workdir_exists): + workdir = cache_tmpdir / pakjoy.CACHE_DIR_NAME + if workdir_exists: + workdir.mkdir() + (workdir / "some_patched_file.pak").ensure() + fake_open = mocker.patch("qutebrowser.misc.pakjoy.open") + + with pakjoy.patch_webengine(): + pass + + assert not fake_open.called + assert not workdir.exists() + + +def test_escape_hatch(affected_version, mocker, monkeypatch): + fake_open = mocker.patch("qutebrowser.misc.pakjoy.open") + monkeypatch.setenv(pakjoy.DISABLE_ENV_VAR, "1") + + with pakjoy.patch_webengine(): + pass + + assert not fake_open.called + + +class TestFindWebengineResources: + @pytest.fixture + def qt_data_path(self, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path): + """Patch qtutils.library_path() to return a temp dir.""" + qt_data_path = tmp_path / "qt_data" + qt_data_path.mkdir() + monkeypatch.setattr(pakjoy.qtutils, "library_path", lambda _which: qt_data_path) + return qt_data_path + + @pytest.fixture + def application_dir_path( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, + qt_data_path: pathlib.Path, # needs patching + ): + """Patch QApplication.applicationDirPath() to return a temp dir.""" + app_dir_path = tmp_path / "app_dir" + app_dir_path.mkdir() + monkeypatch.setattr( + pakjoy.objects.qapp, "applicationDirPath", lambda: app_dir_path + ) + return app_dir_path + + @pytest.fixture + def fallback_path( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, + qt_data_path: pathlib.Path, # needs patching + application_dir_path: pathlib.Path, # needs patching + ): + """Patch the fallback path to return a temp dir.""" + home_path = tmp_path / "home" + monkeypatch.setattr(pakjoy.pathlib.Path, "home", lambda: home_path) + + app_path = home_path / f".{pakjoy.objects.qapp.applicationName()}" + app_path.mkdir(parents=True) + return app_path + + @pytest.mark.parametrize("create_file", [True, False]) + def test_overridden( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path, create_file: bool + ): + """Test the overridden path is used.""" + override_path = tmp_path / "override" + override_path.mkdir() + monkeypatch.setenv(pakjoy.RESOURCES_ENV_VAR, str(override_path)) + if create_file: # should get this no matter if file exists or not + (override_path / pakjoy.PAK_FILENAME).touch() + assert pakjoy._find_webengine_resources() == override_path + + @pytest.mark.parametrize("with_subfolder", [True, False]) + def test_qt_data_path(self, qt_data_path: pathlib.Path, with_subfolder: bool): + """Test qtutils.library_path() is used.""" + resources_path = qt_data_path + if with_subfolder: + resources_path /= "resources" + resources_path.mkdir() + (resources_path / pakjoy.PAK_FILENAME).touch() + assert pakjoy._find_webengine_resources() == resources_path + + def test_application_dir_path(self, application_dir_path: pathlib.Path): + """Test QApplication.applicationDirPath() is used.""" + (application_dir_path / pakjoy.PAK_FILENAME).touch() + assert pakjoy._find_webengine_resources() == application_dir_path + + def test_fallback_path(self, fallback_path: pathlib.Path): + """Test fallback path is used.""" + (fallback_path / pakjoy.PAK_FILENAME).touch() + assert pakjoy._find_webengine_resources() == fallback_path + + def test_nowhere(self, fallback_path: pathlib.Path): + """Test we raise if we can't find the resources.""" + with pytest.raises( + binparsing.ParseError, match="Couldn't find webengine resources dir" + ): + pakjoy._find_webengine_resources() + + +def json_without_comments(bytestring): + str_without_comments = "\n".join( + [ + line + for line in bytestring.decode("utf-8").split("\n") + if not line.strip().startswith("//") + ] + ) + return json.loads(str_without_comments) + + +def read_patched_manifest(): + patched_resources = pathlib.Path(os.environ[pakjoy.RESOURCES_ENV_VAR]) + + with open(patched_resources / pakjoy.PAK_FILENAME, "rb") as fd: + reparsed = pakjoy.PakParser(fd) + + return json_without_comments(reparsed.manifest) + + +@pytest.mark.usefixtures("affected_version") +class TestWithRealResourcesFile: + """Tests that use the real pak file form the Qt installation.""" + + @skip_if_unsupported + def test_happy_path(self): + # Go through the full patching processes with the real resources file from + # the current installation. Make sure our replacement string is in it + # afterwards. + with pakjoy.patch_webengine(): + json_manifest = read_patched_manifest() + + assert ( + pakjoy.REPLACEMENT_URL.decode("utf-8") + in json_manifest["externally_connectable"]["matches"] + ) + + def test_copying_resources(self): + # Test we managed to copy some files over + work_dir = pakjoy.copy_webengine_resources() + + assert work_dir is not None + assert work_dir.exists() + assert work_dir == pathlib.Path(standarddir.cache()) / pakjoy.CACHE_DIR_NAME + assert (work_dir / pakjoy.PAK_FILENAME).exists() + assert len(list(work_dir.glob("*"))) > 1 + + def test_copying_resources_overwrites(self): + work_dir = pakjoy.copy_webengine_resources() + assert work_dir is not None + tmpfile = work_dir / "tmp.txt" + tmpfile.touch() + + pakjoy.copy_webengine_resources() + assert not tmpfile.exists() + + @pytest.mark.parametrize("osfunc", ["copytree", "rmtree"]) + def test_copying_resources_oserror(self, monkeypatch, caplog, osfunc): + # Test errors from the calls to shutil are handled + pakjoy.copy_webengine_resources() # run twice so we hit rmtree too + caplog.clear() + + def raiseme(err): + raise err + + monkeypatch.setattr( + pakjoy.shutil, osfunc, lambda *_args: raiseme(PermissionError(osfunc)) + ) + with caplog.at_level(logging.ERROR, "misc"): + with pakjoy.patch_webengine(): + pass + + assert caplog.messages == [ + "Failed to copy webengine resources, not applying quirk" + ] + + def test_expected_file_not_found(self, cache_tmpdir, monkeypatch, caplog): + with caplog.at_level(logging.ERROR, "misc"): + pakjoy._patch(pathlib.Path(cache_tmpdir) / "doesntexist") + assert caplog.messages[-1].startswith( + "Resource pak doesn't exist at expected location! " + "Not applying quirks. Expected location: " + ) + + +def json_manifest_factory(extension_id=pakjoy.HANGOUTS_MARKER, url=pakjoy.TARGET_URL): + assert isinstance(extension_id, bytes) + assert isinstance(url, bytes) + + return f""" + {{ + {extension_id.decode("utf-8")} + "key": "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDAQt2ZDdPfoSe/JI6ID5bgLHRCnCu9T36aYczmhw/tnv6QZB2I6WnOCMZXJZlRdqWc7w9jo4BWhYS50Vb4weMfh/I0On7VcRwJUgfAxW2cHB+EkmtI1v4v/OU24OqIa1Nmv9uRVeX0GjhQukdLNhAE6ACWooaf5kqKlCeK+1GOkQIDAQAB", + "name": "Google Hangouts", + // Note: Always update the version number when this file is updated. Chrome + // triggers extension preferences update on the version increase. + "version": "1.3.21", + "manifest_version": 2, + "externally_connectable": {{ + "matches": [ + "{url.decode("utf-8")}", + "http://localhost:*/*" + ] + }} + }} + """.strip().encode( + "utf-8" + ) + + +def pak_factory(version=5, entries=None, encoding=1, sentinel_position=-1): + if entries is None: + entries = [json_manifest_factory()] + + buffer = io.BytesIO() + buffer.write(struct.pack("<I", version)) + buffer.write(struct.pack(pakjoy.PakHeader._FORMAT, encoding, len(entries), 0)) + + entry_headers_size = (len(entries) + 1) * 6 + start_of_data = buffer.tell() + entry_headers_size + + # Normally the sentinel sits between the headers and the data. But to get + # full coverage we want to insert it in other positions. + with_indices = list(enumerate(entries, 1)) + if sentinel_position == -1: + with_indices.append((0, b"")) + elif sentinel_position is not None: + with_indices.insert(sentinel_position, (0, b"")) + + accumulated_data_offset = start_of_data + for idx, entry in with_indices: + buffer.write(struct.pack(pakjoy.PakEntry._FORMAT, idx, accumulated_data_offset)) + accumulated_data_offset += len(entry) + + for entry in entries: + assert isinstance(entry, bytes) + buffer.write(entry) + + buffer.seek(0) + return buffer + + +@pytest.mark.usefixtures("affected_version") +class TestWithConstructedResourcesFile: + """Tests that use a constructed pak file to give us more control over it.""" + + @pytest.mark.parametrize( + "offset", + [0, 42, pakjoy.HANGOUTS_ID], # test both slow search and fast path + ) + def test_happy_path(self, offset): + entries = [b""] * offset + [json_manifest_factory()] + assert entries[offset] != b"" + buffer = pak_factory(entries=entries) + + parser = pakjoy.PakParser(buffer) + + json_manifest = json_without_comments(parser.manifest) + + assert ( + pakjoy.TARGET_URL.decode("utf-8") + in json_manifest["externally_connectable"]["matches"] + ) + + def test_bad_version(self): + buffer = pak_factory(version=99) + + with pytest.raises( + binparsing.ParseError, + match="Unsupported .pak version 99", + ): + pakjoy.PakParser(buffer) + + @pytest.mark.parametrize( + "position, error", + [ + (0, "Unexpected sentinel entry"), + (None, "Missing sentinel entry"), + ], + ) + def test_bad_sentinal_position(self, position, error): + buffer = pak_factory(sentinel_position=position) + + with pytest.raises(binparsing.ParseError): + pakjoy.PakParser(buffer) + + @pytest.mark.parametrize( + "entry", + [ + b"{foo}", + b"V2VsbCBoZWxsbyB0aGVyZQo=", + ], + ) + def test_marker_not_found(self, entry): + buffer = pak_factory(entries=[entry]) + + with pytest.raises( + binparsing.ParseError, + match="Couldn't find hangouts manifest", + ): + pakjoy.PakParser(buffer) + + def test_url_not_found(self): + buffer = pak_factory(entries=[json_manifest_factory(url=b"example.com")]) + + parser = pakjoy.PakParser(buffer) + with pytest.raises( + binparsing.ParseError, + match="Couldn't find URL in manifest", + ): + parser.find_patch_offset() + + def test_url_not_found_high_level(self, cache_tmpdir, caplog, affected_version): + buffer = pak_factory(entries=[json_manifest_factory(url=b"example.com")]) + + # Write bytes to file so we can test pakjoy._patch() + tmpfile = pathlib.Path(cache_tmpdir) / "bad.pak" + with open(tmpfile, "wb") as fd: + fd.write(buffer.read()) + + with caplog.at_level(logging.ERROR, "misc"): + pakjoy._patch(tmpfile) + + assert caplog.messages == ["Failed to apply quirk to resources pak."] + + @pytest.fixture + def resources_path( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path + ) -> pathlib.Path: + resources_path = tmp_path / "resources" + resources_path.mkdir() + + buffer = pak_factory() + with open(resources_path / pakjoy.PAK_FILENAME, "wb") as fd: + fd.write(buffer.read()) + + monkeypatch.setattr(pakjoy.qtutils, "library_path", lambda _which: tmp_path) + return resources_path + + @pytest.fixture + def quirk_dir_path(self, tmp_path: pathlib.Path) -> pathlib.Path: + return tmp_path / "cache" / pakjoy.CACHE_DIR_NAME + + def test_patching(self, resources_path: pathlib.Path, quirk_dir_path: pathlib.Path): + """Go through the full patching processes with a fake resources file.""" + with pakjoy.patch_webengine(): + assert os.environ[pakjoy.RESOURCES_ENV_VAR] == str(quirk_dir_path) + json_manifest = read_patched_manifest() + + assert ( + pakjoy.REPLACEMENT_URL.decode("utf-8") + in json_manifest["externally_connectable"]["matches"] + ) + assert pakjoy.RESOURCES_ENV_VAR not in os.environ + + def test_preset_env_var( + self, + resources_path: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, + quirk_dir_path: pathlib.Path, + ): + new_resources_path = resources_path.with_name(resources_path.name + "_moved") + shutil.move(resources_path, new_resources_path) + monkeypatch.setenv(pakjoy.RESOURCES_ENV_VAR, str(new_resources_path)) + + with pakjoy.patch_webengine(): + assert os.environ[pakjoy.RESOURCES_ENV_VAR] == str(quirk_dir_path) + + assert os.environ[pakjoy.RESOURCES_ENV_VAR] == str(new_resources_path) diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 486270d70..38134b40e 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -885,9 +885,7 @@ class TestPDFJSVersion: def test_real_file(self, data_tmpdir): """Test against the real file if pdfjs was found.""" - try: - pdfjs.get_pdfjs_res_and_path('build/pdf.js') - except pdfjs.PDFJSNotFound: + if not pdfjs.is_available(): pytest.skip("No pdfjs found") ver = version._pdfjs_version() assert ver.split()[0] not in ['no', 'unknown'], ver @@ -989,6 +987,17 @@ class TestWebEngineVersions: def test_real_chromium_version(self, qapp): """Compare the inferred Chromium version with the real one.""" + try: + # pylint: disable=unused-import + from qutebrowser.qt.webenginecore import ( + qWebEngineVersion, + qWebEngineChromiumVersion, + ) + except ImportError: + pass + else: + pytest.skip("API available to get the real version") + pyqt_webengine_version = version._get_pyqt_webengine_qt_version() if pyqt_webengine_version is None: if '.dev' in PYQT_VERSION_STR: @@ -51,8 +51,9 @@ deps = pyqt63: -r{toxinidir}/misc/requirements/requirements-pyqt-6.3.txt pyqt64: -r{toxinidir}/misc/requirements/requirements-pyqt-6.4.txt pyqt65: -r{toxinidir}/misc/requirements/requirements-pyqt-6.5.txt + pyqt66: -r{toxinidir}/misc/requirements/requirements-pyqt-6.6.txt commands = - !pyqt-!pyqt515-!pyqt5152-!pyqt62-!pyqt63-!pyqt64-!pyqt65: {envpython} scripts/link_pyqt.py --tox {envdir} + !pyqt-!pyqt515-!pyqt5152-!pyqt62-!pyqt63-!pyqt64-!pyqt65-!pyqt66: {envpython} scripts/link_pyqt.py --tox {envdir} {envpython} -bb -m pytest {posargs:tests} cov: {envpython} scripts/dev/check_coverage.py {posargs} @@ -191,7 +192,6 @@ passenv = APPDATA HOME PYINSTALLER_DEBUG - PYINSTALLER_COMPILE_BOOTLOADER setenv = qt5: PYINSTALLER_QT5=true deps = @@ -281,7 +281,6 @@ passenv = * # Override default PyQt6 from [testenv] setenv = qt5: QUTE_QT_WRAPPER=PyQt5 - PYINSTALLER_COMPILE_BOOTLOADER=true usedevelop = true deps = -r{toxinidir}/requirements.txt |